From 60b1cf6af62b0f54c933ea68b34a6dfb5619aaad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 9 Jul 2022 19:10:29 +0200 Subject: [PATCH 01/79] Doc++ Also don't call CPU features "SIMD", it sounds stupid. --- src/Corrade/Corrade.h | 2 +- src/Corrade/configure.h.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index e14676d2f..6416acc8d 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -534,7 +534,7 @@ direct option and it's only implied by `/arch:AVX`). Superset of @m_since_latest Defined on @ref CORRADE_TARGET_X86 "x86" if -[Supplemental Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSE3) +[Supplemental Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSSE3) are enabled at compile time (on GCC/Clang it's `-mssse3` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE3, implied by @ref CORRADE_TARGET_SSE41. diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 8d7b294d6..d63f3a9e3 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -181,7 +181,7 @@ #define CORRADE_BIG_ENDIAN #endif -/* Compile-time SIMD detection */ +/* Compile-time CPU feature detection */ #ifdef CORRADE_TARGET_X86 /* SSE on GCC: https://stackoverflow.com/a/28939692 */ From 35ef62b6261c3abcbd82deb0e0c8b54ee2041daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 9 Jul 2022 20:14:13 +0200 Subject: [PATCH 02/79] Rethink the CORRADE_TARGET_NEON* defines. Turns out they're for something slightly different than I expected, in particular conversion between 16-bit and 32-bit floats being in basic NEON already, not some extension on top. Which makes sense, and so we don't need a define for it. A separate problem is that the half-floats might be ARM's own variant instead of the IEEE standard, but as that *also* doesn't seem to be a feature available only on certain processors, it doesn't need a define either. Given that half-float conversions are basic NEON, the CORRADE_TARGET_NEON_FMA then is a superse of just CORRADE_TARGET_NEON. It also wasn't checked properly on ARM64 GCC, only on the Android ARM64 Clang, so I fixed that. Then there's actual ARMv8.2-a half-float vector arithmetic, which I don't really need at the moment, but since we have CORRADE_TARGET_NEON_FP16 already, why not keep it but fix it to actually check this feature. It also has FMA builtin, so no need for another CORRADE_TARGET_NEON_FP16_FMA macro controlling just that. The documentation is really bad for the whole ARM ecosystem, with only random PDFs you can't link to, and basically nonexistent Wikipedia presence, so I hope I didn't mess up yet again. --- doc/corrade-changelog.dox | 7 ++++--- src/Corrade/Corrade.h | 35 ++++++++++++++++----------------- src/Corrade/Test/TargetTest.cpp | 14 ++++++------- src/Corrade/configure.h.cmake | 21 +++++++++++--------- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/doc/corrade-changelog.dox b/doc/corrade-changelog.dox index 94e1977f9..da907cd83 100644 --- a/doc/corrade-changelog.dox +++ b/doc/corrade-changelog.dox @@ -48,10 +48,11 @@ namespace Corrade { @ref CORRADE_TARGET_AVX, @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_TARGET_AVX_FMA, @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F, @ref CORRADE_TARGET_NEON, - @ref CORRADE_TARGET_NEON_FP16, @ref CORRADE_TARGET_NEON_FMA and + @ref CORRADE_TARGET_NEON_FMA, @ref CORRADE_TARGET_NEON_FP16 and @ref CORRADE_TARGET_SIMD128 preprocessor variables added to the already - existing @ref CORRADE_TARGET_SSE2 for detecting enabled instruction sets on - x86, ARM and WebAssembly + existing + @ref CORRADE_TARGET_SSE2 for detecting enabled instruction sets on x86, ARM + and WebAssembly - New @ref CORRADE_TARGET_32BIT preprocessor variable for cross-platform detection of 32-bit builds diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 6416acc8d..7208b8ed8 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -274,7 +274,8 @@ unclear on platforms with multi-architecture binaries. If neither nor @ref CORRADE_TARGET_WASM is defined, the platform might be either a very old pre-WebAssembly @ref CORRADE_TARGET_EMSCRIPTEN or any other that the library doesn't know about yet. -@see @ref CORRADE_TARGET_NEON +@see @ref CORRADE_TARGET_NEON, @ref CORRADE_TARGET_NEON_FMA, + @ref CORRADE_TARGET_NEON_FP16 */ #define CORRADE_TARGET_ARM #undef CORRADE_TARGET_ARM @@ -657,7 +658,7 @@ GCC/Clang, `/arch:AVX512` on MSVC). Superset of @ref CORRADE_TARGET_AVX2. Defined on @ref CORRADE_TARGET_ARM "ARM" if [ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) instructions are enabled at compile time (`-mfpu=neon` on GCC/Clang, implicitly -supported on AArch64). Implied by @ref CORRADE_TARGET_NEON_FP16. +supported on ARM64). Implied by @ref CORRADE_TARGET_NEON_FMA. Apart from NEON, there's several other mutually incompatible ARM instruction sets. Detection for these will be added when the platforms become more @@ -673,32 +674,30 @@ sets. Detection for these will be added when the platforms become more #define CORRADE_TARGET_NEON #undef CORRADE_TARGET_NEON -/** -@brief NEON target with half-floats -@m_since_latest - -Defined on @ref CORRADE_TARGET_ARM "ARM" if NEON IEEE -[half-precision floating-point](https://en.wikipedia.org/wiki/Half-precision_floating-point_format) -support is enabled at compile time (`-mfpu=neon-fp16` or higher on GCC/Clang). -Not defined if only the ARM alternative half-float representation is available, -which trades one extra exponent value for a lack of infinity and NaN support. -Superset of @ref CORRADE_TARGET_NEON, implied by @ref CORRADE_TARGET_NEON_FMA. -*/ -#define CORRADE_TARGET_NEON_FP16 -#undef CORRADE_TARGET_NEON_FP16 - /** @brief NEON target with FMA @m_since_latest Defined on @ref CORRADE_TARGET_ARM "ARM" if NEON FMA instructions are enabled -at compile time (`-mfpu=neon-vfpv4` on GCC/Clang). Not defined if FMA is only -available for scalar code and not for NEON. Superset of +at compile time (`-mfpu=neon-vfpv4` on GCC/Clang on 32-bit ARM, implicitly +supported on ARM64). Not defined if FMA is only available for scalar code and +not for NEON. Superset of @ref CORRADE_TARGET_NEON, implied by @ref CORRADE_TARGET_NEON_FP16. */ #define CORRADE_TARGET_NEON_FMA #undef CORRADE_TARGET_NEON_FMA +/** +@brief NEON target with FP16 vector arithmetic +@m_since_latest + +Defined on @ref CORRADE_TARGET_ARM "ARM" if ARMv8.2-a NEON FP16 vector +arithmetic support is enabled at compile time (`-march=armv8.2-a+fp16` on +GCC/Clang). Superset of @ref CORRADE_TARGET_NEON_FMA. +*/ +#define CORRADE_TARGET_NEON_FP16 +#undef CORRADE_TARGET_NEON_FP16 + /** @brief SIMD128 target @m_since_latest diff --git a/src/Corrade/Test/TargetTest.cpp b/src/Corrade/Test/TargetTest.cpp index 1baf5ff77..8e9e65bc6 100644 --- a/src/Corrade/Test/TargetTest.cpp +++ b/src/Corrade/Test/TargetTest.cpp @@ -354,17 +354,17 @@ void TargetTest::simd() { Debug{&out} << "CORRADE_TARGET_NEON"; #endif - #ifdef CORRADE_TARGET_NEON_FP16 - Debug{&out} << "CORRADE_TARGET_NEON_FP16"; + #ifdef CORRADE_TARGET_NEON_FMA + Debug{&out} << "CORRADE_TARGET_NEON_FMA"; #ifndef CORRADE_TARGET_NEON - CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FP16 defined but CORRADE_TARGET_NEON not"); + CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FMA defined but CORRADE_TARGET_NEON not"); #endif #endif - #ifdef CORRADE_TARGET_NEON_FMA - Debug{&out} << "CORRADE_TARGET_NEON_FMA"; - #ifndef CORRADE_TARGET_NEON_FP16 - CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FMA defined but CORRADE_TARGET_NEON_FP16 not"); + #ifdef CORRADE_TARGET_NEON_FP16 + Debug{&out} << "CORRADE_TARGET_NEON_FP16"; + #ifndef CORRADE_TARGET_NEON_FMA + CORRADE_VERIFY(!"CORRADE_TARGET_NEON_FP16 defined but CORRADE_TARGET_NEON_FMA not"); #endif #endif diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index d63f3a9e3..3874af80b 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -256,19 +256,22 @@ #elif defined(CORRADE_TARGET_ARM) #ifdef __ARM_NEON #define CORRADE_TARGET_NEON -/* Conservatively mark half-floats as supported only if the IEEE variant is - supported and not the ARM-specific variant that trades one extra exponent - value for a lack of inf and NaN support (ARM C Language Extensions 1.1, - §6.5.2: https://developer.arm.com/documentation/ihi0053/b/) */ -#if __ARM_FP16_FORMAT_IEEE && (__ARM_NEON_FP & 0x02) -#define CORRADE_TARGET_NEON_FP16 -#endif /* NEON FMA is available only if __ARM_FEATURE_FMA is defined and some bits of __ARM_NEON_FP as well (ARM C Language Extensions 1.1, §6.5.5: - https://developer.arm.com/documentation/ihi0053/b/) */ -#if defined(__ARM_FEATURE_FMA) && __ARM_NEON_FP + https://developer.arm.com/documentation/ihi0053/b/). On AAArch64 NEON is + implicitly supported and __ARM_NEON_FP might not be defined (Android Clang + defines it but GCC 9 on Ubuntu ARM64 not), so check for __aarch64__ as + well. */ +#if defined(__ARM_FEATURE_FMA) && (__ARM_NEON_FP || defined(__aarch64__)) #define CORRADE_TARGET_NEON_FMA #endif +/* There's no linkable documentation for anything and the PDF is stupid. But, + given the FP16 instructions implemented in GCC and Clang are guarded by this + macro, it should be alright: + https://gcc.gnu.org/legacy-ml/gcc-patches/2016-06/msg00460.html */ +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +#define CORRADE_TARGET_NEON_FP16 +#endif #endif /* Undocumented, checked via `echo | em++ -x c++ -dM -E - -msimd128` */ From 6dc6d2917cb01229af768c9439f55c304c357246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 9 Jul 2022 22:28:43 +0200 Subject: [PATCH 03/79] Test: Imply CORRADE_TARGET_WASM, not EMSCRIPTEN from SIMD128. --- src/Corrade/Test/TargetTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Corrade/Test/TargetTest.cpp b/src/Corrade/Test/TargetTest.cpp index 8e9e65bc6..b66455f91 100644 --- a/src/Corrade/Test/TargetTest.cpp +++ b/src/Corrade/Test/TargetTest.cpp @@ -372,12 +372,12 @@ void TargetTest::simd() { CORRADE_VERIFY(!"CORRADE_TARGET_NEON* defined but CORRADE_TARGET_ARM not"); #endif - #ifdef CORRADE_TARGET_EMSCRIPTEN + #ifdef CORRADE_TARGET_WASM #ifdef CORRADE_TARGET_SIMD128 Debug{&out} << "CORRADE_TARGET_SIMD128"; #endif #elif defined(CORRADE_TARGET_SIMD128) - CORRADE_VERIFY(!"CORRADE_TARGET_SIMD128 defined but CORRADE_TARGET_EMSCRIPTEN not"); + CORRADE_VERIFY(!"CORRADE_TARGET_SIMD128 defined but CORRADE_TARGET_WASM not"); #endif Debug{Debug::Flag::NoNewlineAtTheEnd} << out.str(); From ed1c127d3e421af8d7f64d372162267151e0f2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 13 Jul 2022 23:39:59 +0200 Subject: [PATCH 04/79] Rethink how CORRADE_TARGET_AVX_F16C and FMA fit into the hierarchy. In short, they don't. There are various Atom, Celeron, VIA and old Intel Mac chips where it's anything but AVX -> F16C -> FMA -> AVX2. There are some processors with AVX+F16C, some with AVX+FMA, some with AVX+FMA+F16C but not AVX2 and *also* VIA chips with AVX2 but no FMA, as noted in a comment on https://stackoverflow.com/a/50829580. So it's absolutely irresponsible to try to put any order into these. There's none, and anything that wanted to rely on any such order, is broken by design. --- src/Corrade/Corrade.h | 23 ++++++++--------------- src/Corrade/configure.h.cmake | 21 ++++++++++++++++++--- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 7208b8ed8..872090de3 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -587,7 +587,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if [Advanced Vector Extensions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) are enabled at compile time (`-mavx` and higher on GCC/Clang, `/arch:AVX` on MSVC). Superset of @ref CORRADE_TARGET_SSE42, implied by -@ref CORRADE_TARGET_AVX_F16C. +@ref CORRADE_TARGET_AVX2. */ #define CORRADE_TARGET_AVX #undef CORRADE_TARGET_AVX @@ -598,13 +598,8 @@ MSVC). Superset of @ref CORRADE_TARGET_SSE42, implied by Defined on @ref CORRADE_TARGET_X86 "x86" if the [F16C instruction set](https://en.wikipedia.org/wiki/F16C) is enabled at -compile time (`-mf16c` on GCC/Clang, MSVC doesn't have a direct option and it's -only implied by `/arch:AVX2`). Superset of @ref CORRADE_TARGET_AVX, implied by -@ref CORRADE_TARGET_AVX_FMA. - -Although there's no documented relation between AVX, F16C, FMA and AVX2, -looking the history of released Intel and AMD CPUs it can be seen that all CPUs -having F16C support plain AVX as well, and all CPUs supporting FMA have F16C. +compile time. On GCC/Clang it's `-mf16c`, MSVC doesn't have a direct option but +it's assumed to be implied by `/arch:AVX2`. */ #define CORRADE_TARGET_AVX_F16C #undef CORRADE_TARGET_AVX_F16C @@ -614,15 +609,13 @@ having F16C support plain AVX as well, and all CPUs supporting FMA have F16C. @m_since_latest Defined on @ref CORRADE_TARGET_X86 "x86" if the -[FMA3 instruction set](https://en.wikipedia.org/wiki/FMA_instruction_set) is enabled at compile time (`-mfma` on GCC/Clang, MSVC doesn't have a direct -option and it's only implied by `/arch:AVX2`). Superset of -@ref CORRADE_TARGET_AVX_F16C, implied by @ref CORRADE_TARGET_AVX2. +[FMA3 instruction set](https://en.wikipedia.org/wiki/FMA_instruction_set) is +enabled at compile time. On GCC/Clang it's `-mfma`, MSVC doesn't have a direct +option but it's assumes to be implied by `/arch:AVX2`. The FMA4 instruction set, which used to be supported only in certain range of AMD processors and isn't anymore, is not detected, and AMD switched to FMA3 -since. Although there's no documented relation between AVX, F16C, FMA and AVX2, -looking the history of released Intel and AMD CPUs it can be seen that all CPUs -having FMA support F16C as well, and all CPUs supporting AVX2 have FMA. +since. */ #define CORRADE_TARGET_AVX_FMA #undef CORRADE_TARGET_AVX_FMA @@ -634,7 +627,7 @@ having FMA support F16C as well, and all CPUs supporting AVX2 have FMA. Defined on @ref CORRADE_TARGET_X86 "x86" if [Advanced Vector Extensions 2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2) are enabled at compile time (`-mavx2` and higher on GCC/Clang, `/arch:AVX2` on -MSVC). Superset of @ref CORRADE_TARGET_AVX_FMA, implied by +MSVC). Superset of @ref CORRADE_TARGET_AVX, implied by @ref CORRADE_TARGET_AVX512F. */ #define CORRADE_TARGET_AVX2 diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 3874af80b..8b7afb96f 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -185,6 +185,7 @@ #ifdef CORRADE_TARGET_X86 /* SSE on GCC: https://stackoverflow.com/a/28939692 */ +/** @todo check also for Clang, maybe clang-cl defines these too? */ #ifdef CORRADE_TARGET_GCC #ifdef __SSE2__ #define CORRADE_TARGET_SSE2 @@ -232,9 +233,13 @@ #endif #endif -/* On GCC, F16C and FMA have its own define, on MSVC it's implied by /arch:AVX2 - (source: https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86 ... - or at least the FMA instructions, no word about F16C). */ +/* On GCC, F16C and FMA have its own define, on MSVC FMA is implied by + /arch:AVX2 (https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86), + no mention of F16C but https://walbourn.github.io/directxmath-f16c-and-fma/ + says it's like that so I'll believe that. However, comments below + https://stackoverflow.com/a/50829580 say there is a Via processor with AVX2 + but no FMA, so then the __AVX2__ check isn't really bulletproof. Use runtime + detection where possible, please. */ #ifdef CORRADE_TARGET_GCC #ifdef __F16C__ #define CORRADE_TARGET_AVX_F16C @@ -243,9 +248,19 @@ #define CORRADE_TARGET_AVX_FMA #endif #elif defined(CORRADE_TARGET_MSVC) && defined(__AVX2__) +/* As with other AVX+ intrinsics on clang-cl, these are only included if the + corresponding macro is defined as well, so check for __F16C__ and __FMA__ + there. Otherwise this would mean the CORRADE_ENABLE_AVX_{16C,FMA} macros are + defined always, which would incorrectly imply presence of these intrinsics. + Chance is that __F16C__ / __FMA__ is possibly implied with /arch:AVX2, but I + don't know for sure, so this is more robust. */ +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__F16C__) #define CORRADE_TARGET_AVX_F16C +#endif +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__FMA__) #define CORRADE_TARGET_AVX_FMA #endif +#endif /* https://stackoverflow.com/a/37056771, confirmed on Android NDK Clang that __ARM_NEON is indeed still set. For MSVC, according to From 942898af18dc2c4b292282a7ceb9450fbbfa8f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 13 Jul 2022 22:22:09 +0200 Subject: [PATCH 05/79] Add CORRADE_TARGET_POPCNT and CORRADE_TARGET_LZCNT. That may be all CPU features I currently need to detect. --- doc/corrade-changelog.dox | 1 + src/Corrade/Corrade.h | 27 +++++++++++++++++++++++++++ src/Corrade/configure.h.cmake | 29 +++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/doc/corrade-changelog.dox b/doc/corrade-changelog.dox index da907cd83..fd7b62138 100644 --- a/doc/corrade-changelog.dox +++ b/doc/corrade-changelog.dox @@ -45,6 +45,7 @@ namespace Corrade { @ref CORRADE_TARGET_POWERPC - New @ref CORRADE_TARGET_SSE3, @ref CORRADE_TARGET_SSSE3, @ref CORRADE_TARGET_SSE41, @ref CORRADE_TARGET_SSE42, + @ref CORRADE_TARGET_POPCNT, @ref CORRADE_TARGET_LZCNT, @ref CORRADE_TARGET_AVX, @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_TARGET_AVX_FMA, @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F, @ref CORRADE_TARGET_NEON, diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 872090de3..c57a9c1b3 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -579,6 +579,33 @@ doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of #define CORRADE_TARGET_SSE42 #undef CORRADE_TARGET_SSE42 +/** +@brief Target with POPCNT instructions +@m_since_latest + +Defined on @ref CORRADE_TARGET_X86 "x86" if +[POPCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +is enabled at compile time. On GCC/Clang it's `-mpopcnt` and is also implied by +`-msse4.2` and higher, MSVC doesn't have a direct option but it's assumed to +be implied by `/arch:AVX`. +*/ +#define CORRADE_TARGET_POPCNT +#undef CORRADE_TARGET_POPCNT + +/** +@brief Target with LZCNT instructions +@m_since_latest + +Defined on @ref CORRADE_TARGET_X86 "x86" if +[LZCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +is enabled at compile time (on GCC/Clang it's `-mlznct`, MSVC doesn't have a +direct option but it's assumed to be implied by `/arch:AVX`). However note that +this instruction has encoding compatible with an earlier `BSR` instruction +which has a slightly different behavior. +*/ +#define CORRADE_TARGET_LZCNT +#undef CORRADE_TARGET_LZCNT + /** @brief AVX target @m_since_latest diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 8b7afb96f..e22dd75ee 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -233,6 +233,35 @@ #endif #endif +/* POPCNT and LZCNT on GCC, queried wth `gcc -mpopcnt -dM -E - | grep POPCNT`, + and equivalent for LZCNT. */ +#ifdef CORRADE_TARGET_GCC +#ifdef __POPCNT__ +#define CORRADE_TARGET_POPCNT +#endif +#ifdef __LZCNT__ +#define CORRADE_TARGET_LZCNT +#endif + +/* There doesn't seem to be any equivalent on MSVC, + https://github.com/kimwalisch/libpopcnt assumes POPCNT is on x86 MSVC + always, and LZCNT has encoding compatible with BSR so if not available it'll + not crash but produce wrong results, sometimes. Enabling them always feels a + bit too much, so restricting that only to AVX+, since one is in SSE4 + intrinsic headers and the other in AVX headers. */ +#elif defined(CORRADE_TARGET_MSVC) && defined(__AVX__) +#define CORRADE_TARGET_POPCNT +/* As with other AVX+ intrinsics on clang-cl, these are only included if the + corresponding macro is defined as well, so check for __LZCNT__ there. + Otherwise this would mean the CORRADE_ENABLE_LZCNT macro is defined always, + which would incorrectly imply presence of these intrinsics. Chance is that + __LZCNT__ is possibly implied with /arch:AVX2 or some such, but I don't know + for sure, so this is more robust. */ +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__LZCNT__) +#define CORRADE_TARGET_LZCNT +#endif +#endif + /* On GCC, F16C and FMA have its own define, on MSVC FMA is implied by /arch:AVX2 (https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86), no mention of F16C but https://walbourn.github.io/directxmath-f16c-and-fma/ From a339752a4d0626a8a04b200fa60f2aa7938c87dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 10 Jul 2022 01:18:48 +0200 Subject: [PATCH 06/79] New Cpu namespace for compile-time and runtime CPU feature dispatch. ONCE AGAIN the CPU feature detection code has comment:code ratio 1000:1. What a mess. I'm postponing the ARM detection for later, as this is large enough already. Includes also half-float, FMA and AVX-512 tags -- thanks to the extra effort spent researching, all these still form a linear inheritance chain, so no crazy things need to be done. Those will eventually come with the non-foundation AVX-512 instructions, but again maybe that will need a runtime dispatch anyway so it's not such a big deal. Or maybe until we actually use anything, it settles down well enough, with outliers and dead-ends like Xeon Phi being long-abandoned. --- doc/corrade-changelog.dox | 2 + doc/snippets/CMakeLists.txt | 2 + doc/snippets/Corrade-cpp17.cpp | 46 ++ doc/snippets/Corrade.cpp | 68 ++ src/Corrade/CMakeLists.txt | 1 + src/Corrade/Corrade.h | 24 +- src/Corrade/Cpu.cpp | 218 +++++++ src/Corrade/Cpu.h | 956 +++++++++++++++++++++++++++++ src/Corrade/Test/CMakeLists.txt | 2 + src/Corrade/Test/CpuTest.cpp | 461 ++++++++++++++ src/Corrade/Utility/CMakeLists.txt | 2 + src/Corrade/Utility/Memory.h | 8 +- 12 files changed, 1784 insertions(+), 6 deletions(-) create mode 100644 doc/snippets/Corrade-cpp17.cpp create mode 100644 doc/snippets/Corrade.cpp create mode 100644 src/Corrade/Cpu.cpp create mode 100644 src/Corrade/Cpu.h create mode 100644 src/Corrade/Test/CpuTest.cpp diff --git a/doc/corrade-changelog.dox b/doc/corrade-changelog.dox index fd7b62138..584ac8268 100644 --- a/doc/corrade-changelog.dox +++ b/doc/corrade-changelog.dox @@ -56,6 +56,8 @@ namespace Corrade { and WebAssembly - New @ref CORRADE_TARGET_32BIT preprocessor variable for cross-platform detection of 32-bit builds +- New @ref Cpu namespace that provides building blocks for compile-time and + runtime CPU feature detection and dispatch on x86, ARM and WebAssembly @subsubsection corrade-changelog-latest-new-containers Containers library diff --git a/doc/snippets/CMakeLists.txt b/doc/snippets/CMakeLists.txt index 875c0ac96..77daa5f49 100644 --- a/doc/snippets/CMakeLists.txt +++ b/doc/snippets/CMakeLists.txt @@ -38,6 +38,7 @@ set_directory_properties(PROPERTIES CORRADE_USE_PEDANTIC_FLAGS ON) add_library(snippets STATIC + Corrade.cpp Containers.cpp Containers-stl.cpp Utility.cpp) @@ -63,6 +64,7 @@ if(NOT CMAKE_CXX_FLAGS MATCHES "-std=") (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.3") OR (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.10")) add_library(snippets-cpp17 STATIC + Corrade-cpp17.cpp Containers-stl17.cpp) target_link_libraries(snippets-cpp17 PRIVATE CorradeUtility) set_target_properties(snippets-cpp17 PROPERTIES diff --git a/doc/snippets/Corrade-cpp17.cpp b/doc/snippets/Corrade-cpp17.cpp new file mode 100644 index 000000000..688821c89 --- /dev/null +++ b/doc/snippets/Corrade-cpp17.cpp @@ -0,0 +1,46 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Corrade/Cpu.h" +#include "Corrade/Utility/Debug.h" + +#define DOXYGEN_ELLIPSIS(...) __VA_ARGS__ + +using namespace Corrade; + +#ifdef CORRADE_TARGET_X86 +int main() { +/* [Cpu-usage-compile-time] */ +Utility::Debug{} << "Base compiled instruction set:" << Cpu::DefaultBase; + +if constexpr(Cpu::DefaultBase >= Cpu::Avx2) { + // AVX2 code +} else { + // scalar code +} +/* [Cpu-usage-compile-time] */ +} +#endif diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp new file mode 100644 index 000000000..a8026d738 --- /dev/null +++ b/doc/snippets/Corrade.cpp @@ -0,0 +1,68 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Corrade/Cpu.h" +#include "Corrade/Containers/ArrayView.h" +#include "Corrade/Utility/Debug.h" + +#define DOXYGEN_ELLIPSIS(...) __VA_ARGS__ + +using namespace Corrade; + +#ifdef CORRADE_TARGET_X86 +/* [Cpu-usage-declare] */ +void transform(Cpu::ScalarT, Containers::ArrayView data); +void transform(Cpu::Sse42T, Containers::ArrayView data); +void transform(Cpu::Avx2T, Containers::ArrayView data); +/* [Cpu-usage-declare] */ +#endif + +int main() { +#ifdef CORRADE_TARGET_X86 +{ +Containers::ArrayView data; +/* [Cpu-usage-compile-time-call] */ +transform(Cpu::DefaultBase, data); +/* [Cpu-usage-compile-time-call] */ +} + +{ +Containers::ArrayView data; +/* [Cpu-usage-runtime-manual-dispatch] */ +Cpu::Features features = Cpu::runtimeFeatures(); +Utility::Debug{} << "Instruction set available at runtime:" << features; + +if(features & Cpu::Avx2) + transform(Cpu::Avx2, data); +else if(features & Cpu::Sse41) + transform(Cpu::Sse41, data); +else + transform(Cpu::Scalar, data); +/* [Cpu-usage-runtime-manual-dispatch] */ +} +#endif + +} diff --git a/src/Corrade/CMakeLists.txt b/src/Corrade/CMakeLists.txt index 9223e6bb3..18766c818 100644 --- a/src/Corrade/CMakeLists.txt +++ b/src/Corrade/CMakeLists.txt @@ -80,6 +80,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.h.cmake set(Corrade_HEADERS Corrade.h + Cpu.h Tags.h) # Force IDEs to display all header files in project view diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index c57a9c1b3..04434213b 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -256,7 +256,8 @@ library doesn't know about yet. @ref CORRADE_TARGET_SSSE3, @ref CORRADE_TARGET_SSE41, @ref CORRADE_TARGET_SSE42, @ref CORRADE_TARGET_AVX, @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_TARGET_AVX_FMA, - @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F + @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F, + @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_X86 #undef CORRADE_TARGET_X86 @@ -275,7 +276,7 @@ nor @ref CORRADE_TARGET_WASM is defined, the platform might be either a very old pre-WebAssembly @ref CORRADE_TARGET_EMSCRIPTEN or any other that the library doesn't know about yet. @see @ref CORRADE_TARGET_NEON, @ref CORRADE_TARGET_NEON_FMA, - @ref CORRADE_TARGET_NEON_FP16 + @ref CORRADE_TARGET_NEON_FP16, @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_ARM #undef CORRADE_TARGET_ARM @@ -313,7 +314,7 @@ unclear on platforms with multi-architecture binaries. If neither nor @ref CORRADE_TARGET_WASM is defined, the platform might be either a very old pre-WebAssembly @ref CORRADE_TARGET_EMSCRIPTEN or any other that the library doesn't know about yet. -@see @ref CORRADE_TARGET_SIMD128 +@see @ref CORRADE_TARGET_SIMD128, @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_WASM #undef CORRADE_TARGET_WASM @@ -513,6 +514,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if [Streaming SIMD Extensions 2](https://en.wikipedia.org/wiki/SSE2) are enabled at compile time (`-msse2` or higher on GCC/Clang, `/arch:SSE2` or higher on MSVC). All x86-64 targets support SSE2. Implied by @ref CORRADE_TARGET_SSE3. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse2} */ #define CORRADE_TARGET_SSE2 #undef CORRADE_TARGET_SSE2 @@ -526,6 +528,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if at compile time (on GCC/Clang it's `-msse3` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE2, implied by @ref CORRADE_TARGET_SSSE3. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse3} */ #define CORRADE_TARGET_SSE3 #undef CORRADE_TARGET_SSE3 @@ -544,6 +547,7 @@ Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wik but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref CORRADE_TARGET_SSE41 to detect those. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Ssse3} */ #define CORRADE_TARGET_SSSE3 #undef CORRADE_TARGET_SSSE3 @@ -562,6 +566,7 @@ Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wik but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref CORRADE_TARGET_SSE41 to detect those. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse41} */ #define CORRADE_TARGET_SSE41 #undef CORRADE_TARGET_SSE41 @@ -575,6 +580,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if are enabled at compile time (on GCC/Clang it's `-msse4.2` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE41, implied by @ref CORRADE_TARGET_AVX. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse42} */ #define CORRADE_TARGET_SSE42 #undef CORRADE_TARGET_SSE42 @@ -588,6 +594,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if is enabled at compile time. On GCC/Clang it's `-mpopcnt` and is also implied by `-msse4.2` and higher, MSVC doesn't have a direct option but it's assumed to be implied by `/arch:AVX`. +@see @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_POPCNT #undef CORRADE_TARGET_POPCNT @@ -602,6 +609,7 @@ is enabled at compile time (on GCC/Clang it's `-mlznct`, MSVC doesn't have a direct option but it's assumed to be implied by `/arch:AVX`). However note that this instruction has encoding compatible with an earlier `BSR` instruction which has a slightly different behavior. +@see @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_LZCNT #undef CORRADE_TARGET_LZCNT @@ -615,6 +623,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if are enabled at compile time (`-mavx` and higher on GCC/Clang, `/arch:AVX` on MSVC). Superset of @ref CORRADE_TARGET_SSE42, implied by @ref CORRADE_TARGET_AVX2. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx} */ #define CORRADE_TARGET_AVX #undef CORRADE_TARGET_AVX @@ -627,6 +636,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if the [F16C instruction set](https://en.wikipedia.org/wiki/F16C) is enabled at compile time. On GCC/Clang it's `-mf16c`, MSVC doesn't have a direct option but it's assumed to be implied by `/arch:AVX2`. +@see @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_AVX_F16C #undef CORRADE_TARGET_AVX_F16C @@ -643,6 +653,7 @@ option but it's assumes to be implied by `/arch:AVX2`. The FMA4 instruction set, which used to be supported only in certain range of AMD processors and isn't anymore, is not detected, and AMD switched to FMA3 since. +@see @relativeref{Corrade,Cpu} */ #define CORRADE_TARGET_AVX_FMA #undef CORRADE_TARGET_AVX_FMA @@ -656,6 +667,7 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if are enabled at compile time (`-mavx2` and higher on GCC/Clang, `/arch:AVX2` on MSVC). Superset of @ref CORRADE_TARGET_AVX, implied by @ref CORRADE_TARGET_AVX512F. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx2} */ #define CORRADE_TARGET_AVX2 #undef CORRADE_TARGET_AVX2 @@ -667,6 +679,7 @@ MSVC). Superset of @ref CORRADE_TARGET_AVX, implied by Defined on @ref CORRADE_TARGET_X86 "x86" if [AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation instructions are enabled at compile time (`-mavx512f` and higher on GCC/Clang, `/arch:AVX512` on MSVC). Superset of @ref CORRADE_TARGET_AVX2. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx512f} */ #define CORRADE_TARGET_AVX512F #undef CORRADE_TARGET_AVX512F @@ -690,6 +703,8 @@ sets. Detection for these will be added when the platforms become more - SVE2, which is a next-generation vector instruction set designed to be a successor to both NEON and SVE, scheduled to appear in production in around 2022 + +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Neon} */ #define CORRADE_TARGET_NEON #undef CORRADE_TARGET_NEON @@ -703,6 +718,7 @@ at compile time (`-mfpu=neon-vfpv4` on GCC/Clang on 32-bit ARM, implicitly supported on ARM64). Not defined if FMA is only available for scalar code and not for NEON. Superset of @ref CORRADE_TARGET_NEON, implied by @ref CORRADE_TARGET_NEON_FP16. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFma} */ #define CORRADE_TARGET_NEON_FMA #undef CORRADE_TARGET_NEON_FMA @@ -714,6 +730,7 @@ not for NEON. Superset of @ref CORRADE_TARGET_NEON, implied by Defined on @ref CORRADE_TARGET_ARM "ARM" if ARMv8.2-a NEON FP16 vector arithmetic support is enabled at compile time (`-march=armv8.2-a+fp16` on GCC/Clang). Superset of @ref CORRADE_TARGET_NEON_FMA. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFp16} */ #define CORRADE_TARGET_NEON_FP16 #undef CORRADE_TARGET_NEON_FP16 @@ -725,6 +742,7 @@ GCC/Clang). Superset of @ref CORRADE_TARGET_NEON_FMA. Defined on @ref CORRADE_TARGET_WASM "WebAssembly" if [128-bit SIMD](https://github.com/webassembly/simd) instructions are enabled at compile time (`-msimd128` passed to Clang). +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Simd128} */ #define CORRADE_TARGET_SIMD128 #undef CORRADE_TARGET_SIMD128 diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp new file mode 100644 index 000000000..2b8cf58ab --- /dev/null +++ b/src/Corrade/Cpu.cpp @@ -0,0 +1,218 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Cpu.h" + +#include "Corrade/Containers/StringView.h" +#include "Corrade/Utility/Debug.h" +#include "Corrade/Utility/DebugAssert.h" + +#ifdef CORRADE_TARGET_X86 +#include +#ifdef CORRADE_TARGET_MSVC +#include +#include +#elif defined(CORRADE_TARGET_GCC) +#include +/* _xgetbv() defined since GCC 8.1, in , but I can't include + that one alone, have to include instead: + https://github.com/gcc-mirror/gcc/commit/b9bdd60b8792e2d3173ecfacd5c25aac894a94e5 + On Clang it got first defined in 3.5: + https://github.com/llvm/llvm-project/commit/854f7d34ec3d7c0559ffde536cfa88fad5419bf0 + but that's (look again) in Intrin.h (yes, uppercase), which got fixed to + be intrin.h in 3.9, however at least on Xcode 9.4 intrin.h has + `#include_next ` and there *just isn't* any other intrin.h so + it fails; then, looking at the commit history, the function changed + locations a few times, mostly as an attempt to fix compilation of V8 (?!?!) + and ... I have no patience for this anymore, so for Clang I'm using the same + inline assembly as for GCC 8.0 and earlier. */ +#if defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 801 +#include +#endif +#endif +#endif + +namespace Corrade { namespace Cpu { + +using namespace Containers::Literals; + +/* As the types all inherit from each other, there should be no members to + keep them zero-cost. */ +static_assert(sizeof(Cpu::Scalar) == 1, ""); +#ifdef CORRADE_TARGET_X86 +static_assert(sizeof(Cpu::Sse2) == 1, ""); +static_assert(sizeof(Cpu::Sse3) == 1, ""); +static_assert(sizeof(Cpu::Ssse3) == 1, ""); +static_assert(sizeof(Cpu::Sse41) == 1, ""); +static_assert(sizeof(Cpu::Sse42) == 1, ""); +static_assert(sizeof(Cpu::Avx) == 1, ""); +static_assert(sizeof(Cpu::Avx2) == 1, ""); +static_assert(sizeof(Cpu::Avx512f) == 1, ""); +#elif defined(CORRADE_TARGET_ARM) +static_assert(sizeof(Cpu::Neon) == 1, ""); +static_assert(sizeof(Cpu::NeonFma) == 1, ""); +static_assert(sizeof(Cpu::NeonFp16) == 1, ""); +#elif defined(CORRADE_TARGET_WASM) +static_assert(sizeof(Cpu::Simd128) == 1, ""); +#endif + +#if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) +#if defined(CORRADE_TARGET_X86) && defined(CORRADE_TARGET_GCC) +/* GCC 10 complains otherwise, however 4.8 doesn't. Doing it via a function + attribute instead of passing -mxsave as detecting target architecture in + CMake is annoying at best. */ +__attribute__((__target__("xsave"))) +#endif +Features runtimeFeatures() { + unsigned int out = 0; + + /* Use cpuid on x86 and GCC/Clang/MSVC */ + #if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) + union { + struct { + std::uint32_t ax, bx, cx, dx; + } e; + #ifdef CORRADE_TARGET_MSVC + /* MSVC's __cpuid() macro accepts an array instead of four pointers and + wants an int, not uint, for some reason */ + std::int32_t data[4]; + #endif + } cpuid{}; + + /* https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits */ + #ifdef CORRADE_TARGET_MSVC + __cpuid(cpuid.data, 1); + #elif defined(CORRADE_TARGET_GCC) + CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid(1, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); + #endif + + if(cpuid.e.dx & (1 << 26)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 0)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 9)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 19)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 20)) out |= TypeTraits::Index; + + /* AVX needs OS support checked, as the OS needs to be capable of saving + and restoring the expanded registers when switching contexts: + https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Operating_system_support */ + if((cpuid.e.cx & (1 << 27)) && /* XSAVE/XRESTORE CPU support */ + (cpuid.e.cx & (1 << 28))) /* AVX CPU support */ + { + /* XGETBV indicates that the registers will be properly saved and + restored by the OS: https://stackoverflow.com/a/22521619. The + _xgetbv() function is available on MSVC and since GCC 8.1 and + probably on Clang also but I don't care -- see the comment at the + #include above. */ + #if defined(CORRADE_TARGET_MSVC) || (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 801) + const bool available = (_xgetbv(0) & 0x6) == 0x6; + #else + /* https://github.com/vectorclass/version2/blob/ff7450acfad9d3a7c6825d92cfb782a42ccfa71f/instrset_detect.cpp#L30-L32 */ + std::uint32_t a, d; + __asm("xgetbv" : "=a"(a),"=d"(d) : "c"(0) : ); + const bool available = ((a|(std::uint64_t(d) << 32)) & 0x6) == 0x6; + #endif + + if(available) out |= TypeTraits::Index; + } + + /* If AVX is not supported, we don't check any following flags either */ + if(!(out & TypeTraits::Index)) return Features{out}; + + /* https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features */ + #ifdef CORRADE_TARGET_MSVC + __cpuidex(cpuid.data, 7, 0); + /* __get_cpuid_count() is only since GCC 7.1 and Clang 5.0: + https://github.com/gcc-mirror/gcc/commit/2488ebe5ef1788616c2fbc61e05af09f0749ebbe + https://github.com/llvm/llvm-project/commit/f6e8408a116405d0cc25b506e8c5b60ab4ab7bcd + Furthermore, Clang 5.0 is 9.3 on Apple, 9.2 has only Clang 4: + https://en.wikipedia.org/wiki/Xcode#Toolchain_versions */ + #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 701) || (defined(CORRADE_TARGET_CLANG) && ((!defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__ >= 5) || (defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__*100 + __clang_minor__ >= 903))) + CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid_count(7, 0, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); + /* Looking at the above commits, on older GCC, __get_cpuid(7) seems to do + what __get_cpuid_count(7, 0) does on new GCC, however that's only since + 6.3: + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 + Older Clang doesn't do any of that, and thus to avoid a combinatorial + explosion of test scenarios, we'll just directly use the __cpuid + assembly on both, which seems to be there since forever. + __get_cpuid[_count]() only does a bunch of bounds checking on top of + that anyway and if we didn't blow up on the assert above I HOPE we + should be fine to check for extended features level zero. For higher + levels (to check very recent AVX512 features) probably not but I assume + those would need very recent compilers anyway so when that happens we + can just pretend those don't exist on GCC 7.0 and older. */ + #else + __cpuid_count(7, 0, cpuid.e.ax, cpuid.e.bx, cpuid.e.cx, cpuid.e.dx); + #endif + if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; + if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; + + /* No other implementation at the moment. The function should not be even + defined here in that case -- it's inlined in the header instead. */ + #else + #error + #endif + + return Features{out}; +} +#endif + +Utility::Debug& operator<<(Utility::Debug& debug, const Features value) { + const Containers::StringView prefix = "|Cpu::"_s.exceptSuffix(debug.immediateFlags() & Utility::Debug::Flag::Packed ? 5 : 0); + + /* First one without the | */ + debug << prefix.exceptPrefix(1) << Utility::Debug::nospace; + if(!value) return debug << "Scalar"_s; + + bool written = false; + #define _c(tag) \ + if(value & tag) { \ + if(!written) written = true; \ + else debug << Utility::Debug::nospace << prefix << Utility::Debug::nospace; \ + debug << TypeTraits::name(); \ + } + #ifdef CORRADE_TARGET_X86 + _c(Sse2) + _c(Sse3) + _c(Ssse3) + _c(Sse41) + _c(Sse42) + _c(Avx) + _c(Avx2) + _c(Avx512f) + #elif defined(CORRADE_TARGET_ARM) + _c(Neon) + _c(NeonFma) + _c(NeonFp16) + #elif defined(CORRADE_TARGET_EMSCRIPTEN) + _c(Simd128) + #endif + #undef _c + + return debug; +} + +}} diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h new file mode 100644 index 000000000..5286006f8 --- /dev/null +++ b/src/Corrade/Cpu.h @@ -0,0 +1,956 @@ +#ifndef Corrade_Cpu_h +#define Corrade_Cpu_h +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file + * @brief Namespace @ref Corrade::Cpu + * @m_since_latest + */ + +#include "Corrade/Utility/Utility.h" +#include "Corrade/Utility/visibility.h" + +namespace Corrade { + +/** +@brief Compile-time and runtime CPU instruction set detection and dispatch +@m_since_latest + +This namespace provides *tags* for x86, ARM and WebAssembly instruction sets, +which can be used for either system introspection or for choosing a particular +implementation based on the available instruction set. These tags build on top +of the @ref CORRADE_TARGET_SSE2, @ref CORRADE_TARGET_SSE3 etc. preprocessor +macros and provide a runtime feature detection as well. + +This library is built if `WITH_UTILITY` is enabled when building Corrade. To +use this library with CMake, request the `Utility` component of the `Corrade` +package and link to the `Corrade::Utility` target: + +@code{.cmake} +find_package(Corrade REQUIRED Utility) + +# ... +target_link_libraries(your-app PRIVATE Corrade::Utility) +@endcode + +@section Cpu-usage Usage + +The @ref Cpu namespace contains tags such as @ref Cpu::Avx2, @ref Cpu::Sse2, +@ref Cpu::Neon or @ref Cpu::Simd128. These tags behave similarly to enum values +and their combination result in @ref Cpu::Features, which is similar to the +@ref Containers::EnumSet class --- they support the same bitwise operations, +can be tested for subsets and supersets, and are printable with +@ref Utility::Debug. + +The most advanced base CPU instruction set enabled at compile time is then +exposed through the @ref Cpu::DefaultBase variable, which is an alias to one of +those tags, and it matches the architecture-specific @ref CORRADE_TARGET_SSE2 +etc. macros. Since it's a @cpp constexpr @ce variable, it's usable in a +compile-time context. The most straightforward use is shown in the following +C++17 snippet: + +@snippet Corrade-cpp17.cpp Cpu-usage-compile-time + + + +@m_class{m-note m-info} + +@par + If you're writing multiplatform code targeting multiple architectures, you + still need to partially rely on the preprocessor when using the + architecture-specific tags, as those are defined only on the architecture + they apply to. The above would need to be wrapped in + @cpp #ifdef CORRADE_TARGET_X86 @ce; if you would be checking for + @ref Cpu::Neon instead, then you'd need to wrap it in a + @ref CORRADE_TARGET_ARM check. On the other hand, the per-architecture tags + are available on given architecture always --- so for example + @ref Cpu::Avx512f is present even on a compiler that doesn't even recognize + AVX-512 yet. + +@subsection Cpu-usage-dispatch-compile-time Dispatching on available CPU instruction set at compile time + +The main purpose of these tags, however, is to provide means for a compile-time +overload resolution. In other words, picking the best candidate among a set of +functions implemented with various instruction sets. As an example, let's say +you have three different implementations of a certain algorithm transforming +numeric data. One is using AVX2 instructions, another is a slower variant using +just SSE 4.2 and as a fallback there's one with just regular scalar code. To +distinguish them, the functions have the same name, but use a different *tag +type*: + +@snippet Corrade.cpp Cpu-usage-declare + +Then you can either call a particular implementation directly --- for example +to test it --- or you can pass @ref Cpu::DefaultBase, and it'll pick the best +overload candidate for the set of CPU instruction features enabled at compile +time: + +@snippet Corrade.cpp Cpu-usage-compile-time-call + +- If the user code was compiled with AVX2 or higher enabled, the + @ref Cpu::Avx2 overload will be picked. +- Otherwise, if just AVX, SSE 4.2 or anything else that includes SSE 4.2 was + enabled, the @ref Cpu::Sse42 overload will be picked. +- Otherwise (for example when compiling for generic x86-64 that has just the + SSE2 feature set), the @ref Cpu::Scalar overload will be picked. If you + wouldn't provide this overload, the compilation would fail for such a + target --- which is useful for example to enforce a certain CPU feature set + to be enabled in order to use a certain API. + + + +@m_class{m-block m-warning} + +@par SSE3, SSSE3 and SSE4.1/SSE4.2 on MSVC + A special case worth mentioning are SSE3 and newer instructions on Windows. + MSVC only provides a very coarse `/arch:SSE2`, `/arch:AVX` and `/arch:AVX2` + for either @ref Sse2, @ref Avx or @ref Avx2, but nothing in between. That + means it's impossible to rely just on compile-time detection to use the + later SSE features on machines that don't support AVX yet (or the various + AVX additions on machines without AVX2), you have to use runtime dispatch + there, as shown below. + +@subsection Cpu-usage-dispatch-runtime Runtime detection and manual dispatch + +So far that was all compile-time detection, which has use mainly when a binary +can be optimized directly for the machine it will run on. But such approach is +not practical when shipping to a heterogenous set of devices. Instead, the +usual workflow is that the majority of code uses the lowest common +denominator (such as SSE2 on x86), with the most demanding functions having +alternative implementations --- picked at runtime --- that make use of more +advanced instructions for better performance. + +Runtime detection is exposed through @ref Cpu::runtimeFeatures(). It will +detect CPU features on platforms that support it, and fall back to +@ref Cpu::compiledFeatures() on platforms that don't. You can then match the +returned @ref Cpu::Features against particular tags to decide which variant to +use: + +@snippet Corrade.cpp Cpu-usage-runtime-manual-dispatch +*/ +namespace Cpu { + +/** +@brief Traits class for CPU detection tag types + +Useful for detecting tag properties at compile time without the need for +repeated code such as method overloading, cascaded ifs or template +specializations for all tag types. All tag types in the @ref Cpu namespace +have this class implemented. +*/ +#ifndef DOXYGEN_GENERATING_OUTPUT +template struct TypeTraits; +#else +template struct TypeTraits { + enum: unsigned int { + /** + * Tag-specific index. Implementation-defined, is unique among all tags + * on given platform. + */ + Index + }; + + /** + * @brief Tag name + * + * Returns a string representation of the tag, such as @cpp "Avx2" @ce + * for @ref Avx2. + */ + static const char* name(); +}; +#endif + +namespace Implementation { + /* A common type used in all tag constructors to avoid ambiguous calls when + using {} */ + struct InitT {}; + constexpr InitT Init{}; +} + +/** +@brief Scalar tag type + +See the @ref Cpu namespace and the @ref Scalar tag for more information. +*/ +struct ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit ScalarT(Implementation::InitT) {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Scalar code is when nothing else is available, thus no bits set */ +template<> struct TypeTraits { + enum: unsigned int { Index = 0 }; + static const char* name() { return "Scalar"; } +}; +#endif + +#if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SSE2 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse2 tag for more information. +*/ +struct Sse2T: ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse2T(Implementation::InitT): ScalarT{Implementation::Init} {} + #endif +}; + +/** +@brief SSE3 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse3 tag for more information. +*/ +struct Sse3T: Sse2T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse3T(Implementation::InitT): Sse2T{Implementation::Init} {} + #endif +}; + +/** +@brief SSSE3 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Ssse3 tag for more information. +*/ +struct Ssse3T: Sse3T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Ssse3T(Implementation::InitT): Sse3T{Implementation::Init} {} + #endif +}; + +/** +@brief SSE4.1 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse41T tag for more information. +*/ +struct Sse41T: Ssse3T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse41T(Implementation::InitT): Ssse3T{Implementation::Init} {} + #endif +}; + +/** +@brief SSE4.2 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Sse42T tag for more information. +*/ +struct Sse42T: Sse41T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Sse42T(Implementation::InitT): Sse41T{Implementation::Init} {} + #endif +}; + +/** +@brief AVX tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Avx tag for more information. +*/ +struct AvxT: Sse42T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit AvxT(Implementation::InitT): Sse42T{Implementation::Init} {} + #endif +}; + +/** +@brief AVX2 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Avx2 tag for more information. +*/ +struct Avx2T: AvxT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Avx2T(Implementation::InitT): AvxT{Implementation::Init} {} + #endif +}; + +/** +@brief AVX-512 Foundation tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Avx512f tag for more information. +*/ +struct Avx512fT: Avx2T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Avx512fT(Implementation::InitT): Avx2T{Implementation::Init} {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 0 }; + static const char* name() { return "Sse2"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 1 }; + static const char* name() { return "Sse3"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 2 }; + static const char* name() { return "Ssse3"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 3 }; + static const char* name() { return "Sse41"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 4 }; + static const char* name() { return "Sse42"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 5 }; + static const char* name() { return "Avx"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 6 }; + static const char* name() { return "Avx2"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 7 }; + static const char* name() { return "Avx512f"; } +}; +#endif +#endif + +#if defined(CORRADE_TARGET_ARM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief NEON tag type + +Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace +and the @ref Neon tag for more information. +*/ +struct NeonT: ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit NeonT(Implementation::InitT): ScalarT{Implementation::Init} {} + #endif +}; + +/** +@brief NEON FMA tag type + +Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace +and the @ref NeonFma tag for more information. +*/ +struct NeonFmaT: NeonT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit NeonFmaT(Implementation::InitT): NeonT{Implementation::Init} {} + #endif +}; + +/** +@brief NEON FP16 tag type + +Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace +and the @ref NeonFp16 tag for more information. +*/ +struct NeonFp16T: NeonFmaT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit NeonFp16T(Implementation::InitT): NeonFmaT{Implementation::Init} {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 0 }; + static const char* name() { return "Neon"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 1 }; + static const char* name() { return "NeonFma"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 2 }; + static const char* name() { return "NeonFp16"; } +}; +#endif +#endif + +#if defined(CORRADE_TARGET_WASM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SIMD128 tag type + +Available only on @ref CORRADE_TARGET_WASM "WebAssembly". See the @ref Cpu +namespace and the @ref Simd128 tag for more information. +*/ +struct Simd128T: ScalarT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Simd128T(Implementation::InitT): ScalarT{Implementation::Init} {} + #endif +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << 0 }; + static const char* name() { return "Simd128"; } +}; +#endif +#endif + +/** +@brief Scalar tag + +Code that isn't explicitly optimized with any advanced CPU instruction set. +Fallback if no other CPU instruction set is chosen or available. The next most +widely supported instruction sets are @ref Sse2 on x86, @ref Neon on ARM and +@ref Simd128 on WebAssembly. +*/ +constexpr ScalarT Scalar{Implementation::Init}; + +#if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SSE2 tag + +[Streaming SIMD Extensions 2](https://en.wikipedia.org/wiki/SSE2). Available +only on @ref CORRADE_TARGET_X86 "x86", supported by all 64-bit x86 processors +and is present on majority of contemporary 32-bit x86 processors as well. +Superset of @ref Scalar, implied by @ref Sse3. +@see @ref CORRADE_TARGET_SSE2 +*/ +constexpr Sse2T Sse2{Implementation::Init}; + +/** +@brief SSE3 tag + +[Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSE3). Available +only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse2, implied by +@ref Ssse3. +@see @ref CORRADE_TARGET_SSE3 +*/ +constexpr Sse3T Sse3{Implementation::Init}; + +/** +@brief SSSE3 tag + +[Supplemental Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSSE3). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse3, implied +by @ref Sse41. + +Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wiki/SSE4#SSE4a) +but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 +to a large extent, and it's recommended to use @ref Sse41 to handle those. +@see @ref CORRADE_TARGET_SSSE3 +*/ +constexpr Ssse3T Ssse3{Implementation::Init}; + +/** +@brief SSE4.1 tag + +[Streaming SIMD Extensions 4.1](https://en.wikipedia.org/wiki/SSE4#SSE4.1). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Ssse3, +implied by @ref Sse42. + +Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wiki/SSE4#SSE4a) +but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 +to a large extent, and it's recommended to use @ref Sse41 to handle those. +@see @ref CORRADE_TARGET_SSE41 +*/ +constexpr Sse41T Sse41{Implementation::Init}; + +/** +@brief SSE4.2 tag + +[Streaming SIMD Extensions 4.2](https://en.wikipedia.org/wiki/SSE4#SSE4.2). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse41, +implied by @ref Avx. +@see @ref CORRADE_TARGET_SSE42 +*/ +constexpr Sse42T Sse42{Implementation::Init}; + +/** +@brief AVX tag + +[Advanced Vector Extensions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse42, +implied by @ref Avx2. +@see @ref CORRADE_TARGET_AVX +*/ +constexpr AvxT Avx{Implementation::Init}; + +/** +@brief AVX2 tag + +[Advanced Vector Extensions 2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2). +Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Avx, +implied by @ref Avx512f. +@see @ref CORRADE_TARGET_AVX2 +*/ +constexpr Avx2T Avx2{Implementation::Init}; + +/** +@brief AVX-512 Foundation tag + +[AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation. Available only on +@ref CORRADE_TARGET_X86 "x86". Superset of @ref Avx2. +@see @ref CORRADE_TARGET_AVX512F +*/ +constexpr Avx512fT Avx512f{Implementation::Init}; +#endif + +#if defined(CORRADE_TARGET_ARM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief NEON tag type + +[ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)). +Available only on @ref CORRADE_TARGET_ARM "ARM". Superset of @ref Scalar, +implied by @ref NeonFp16. +@see @ref CORRADE_TARGET_NEON +*/ +constexpr NeonT Neon{Implementation::Init}; + +/** +@brief NEON FMA tag type + +[ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +with FMA instructions. Available only on @ref CORRADE_TARGET_ARM "ARM". +Superset of @ref Neon, implied by @ref NeonFp16. +@see @ref CORRADE_TARGET_NEON_FMA +*/ +constexpr NeonFmaT NeonFma{Implementation::Init}; + +/** +@brief NEON FP16 tag type + +[ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +with ARMv8.2-a FP16 vector arithmetic. Available only on +@ref CORRADE_TARGET_ARM "ARM". Superset of @ref NeonFma. +@see @ref CORRADE_TARGET_NEON_FP16 +*/ +constexpr NeonFp16T NeonFp16{Implementation::Init}; +#endif + +#if defined(CORRADE_TARGET_WASM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief SIMD128 tag type + +[128-bit WebAssembly SIMD](https://github.com/webassembly/simd). Available only +on @ref CORRADE_TARGET_WASM "WebAssembly". Superset of @ref Scalar. +@see @ref CORRADE_TARGET_SIMD128 +*/ +constexpr Simd128T Simd128{Implementation::Init}; +#endif + +/** +@brief Default base tag type + +See the @ref DefaultBase tag for more information. +*/ +typedef + #ifdef CORRADE_TARGET_X86 + #ifdef CORRADE_TARGET_AVX512F + Avx512fT + #elif defined(CORRADE_TARGET_AVX2) + Avx2T + #elif defined(CORRADE_TARGET_AVX) + AvxT + #elif defined(CORRADE_TARGET_SSE42) + Sse42T + #elif defined(CORRADE_TARGET_SSE41) + Sse41T + #elif defined(CORRADE_TARGET_SSSE3) + Ssse3T + #elif defined(CORRADE_TARGET_SSE3) + Sse3T + #elif defined(CORRADE_TARGET_SSE2) + Sse2T + #else + ScalarT + #endif + + #elif defined(CORRADE_TARGET_ARM) + #ifdef CORRADE_TARGET_NEON_FP16 + NeonFp16T + #elif defined(CORRADE_TARGET_NEON_FMA) + NeonFmaT + #elif defined(CORRADE_TARGET_NEON) + NeonT + #else + ScalarT + #endif + + #elif defined(CORRADE_TARGET_WASM) + #ifdef CORRADE_TARGET_SIMD128 + Simd128T + #else + ScalarT + #endif + #endif + DefaultBaseT; + +/** +@brief Default base tag + +Highest base instruction set available on given architecture with current +compiler flags. Ordered by priority, on @ref CORRADE_TARGET_X86 it's one of these: + +- @ref Avx512f if @ref CORRADE_TARGET_AVX512F is defined +- @ref Avx2 if @ref CORRADE_TARGET_AVX2 is defined +- @ref Avx if @ref CORRADE_TARGET_AVX is defined +- @ref Sse42 if @ref CORRADE_TARGET_SSE42 is defined +- @ref Sse41 if @ref CORRADE_TARGET_SSE41 is defined +- @ref Ssse3 if @ref CORRADE_TARGET_SSSE3 is defined +- @ref Sse3 if @ref CORRADE_TARGET_SSE3 is defined +- @ref Sse2 if @ref CORRADE_TARGET_SSE2 is defined +- @ref Scalar otherwise + +On @ref CORRADE_TARGET_ARM it's one of these: + +- @ref NeonFp16 if @ref CORRADE_TARGET_NEON_FP16 is defined +- @ref NeonFma if @ref CORRADE_TARGET_NEON_FMA is defined +- @ref Neon if @ref CORRADE_TARGET_NEON is defined +- @ref Scalar otherwise + +On @ref CORRADE_TARGET_WASM it's one of these: + +- @ref Simd128 if @ref CORRADE_TARGET_SIMD128 is defined +- @ref Scalar otherwise + +See also @ref compiledFeatures(), which returns a *combination* of these tags +instead of just the highest available, and @ref runtimeFeatures() which is +capable of detecting the available CPU feature set at runtime. +*/ +constexpr DefaultBaseT DefaultBase{Implementation::Init}; + +/** +@brief Feature set + +Provides storage and comparison as well as runtime detection of CPU instruction +set. Provides an interface similar to an @ref Containers::EnumSet, with values +being the @ref Sse2, @ref Sse3 etc. tags. + +See the @ref Cpu namespace for an overview and usage examples. +@see @ref compiledFeatures(), @ref runtimeFeatures() +*/ +class Features { + public: + /** + * @brief Default constructor + * + * Equivalent to @ref Scalar. + */ + constexpr explicit Features() noexcept: _data{} {} + + /** @brief Construct from a tag */ + template::Index)> constexpr /*implicit*/ Features(T) noexcept: _data{TypeTraits::Index} {} + + /** @brief Equality comparison */ + constexpr bool operator==(Features other) const { + return _data == other._data; + } + + /** @brief Non-equality comparison */ + constexpr bool operator!=(Features other) const { + return _data != other._data; + } + + /** + * @brief Whether @p other is a subset of this (@f$ a \supseteq o @f$) + * + * Equivalent to @cpp (a & other) == other @ce. + */ + constexpr bool operator>=(Features other) const { + return (_data & other._data) == other._data; + } + + /** + * @brief Whether @p other is a superset of this (@f$ a \subseteq o @f$) + * + * Equivalent to @cpp (a & other) == a @ce. + */ + constexpr bool operator<=(Features other) const { + return (_data & other._data) == _data; + } + + /** @brief Union of two feature sets */ + constexpr Features operator|(Features other) const { + return Features{_data | other._data}; + } + + /** @brief Union two feature sets and assign */ + Features& operator|=(Features other) { + _data |= other._data; + return *this; + } + + /** @brief Intersection of two feature sets */ + constexpr Features operator&(Features other) const { + return Features{_data & other._data}; + } + + /** @brief Intersect two feature sets and assign */ + Features& operator&=(Features other) { + _data &= other._data; + return *this; + } + + /** @brief XOR of two feature sets */ + constexpr Features operator^(Features other) const { + return Features{_data ^ other._data}; + } + + /** @brief XOR two feature sets and assign */ + Features& operator^=(Features other) { + _data ^= other._data; + return *this; + } + + /** @brief Feature set complement */ + constexpr Features operator~() const { + return Features{~_data}; + } + + /** + * @brief Boolean conversion + * + * Returns @cpp true @ce if at least one feature apart from @ref Scalar + * is present, @cpp false @ce otherwise. + */ + constexpr explicit operator bool() const { return _data; } + + /** + * @brief Integer representation + * + * For testing purposes. @ref Cpu::Scalar is always @cpp 0 @ce, values + * corresponding to other feature tags are unspecified. + */ + constexpr explicit operator unsigned int() const { return _data; } + + private: + friend constexpr Features compiledFeatures(); + #if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) + /* MSVC demands the export macro to be here as well */ + friend CORRADE_UTILITY_EXPORT Features runtimeFeatures(); + #endif + + constexpr explicit Features(unsigned int data) noexcept: _data{data} {} + + unsigned int _data; +}; + +/** @relates Features +@brief Equality comparison of a tag and a feature set + +Same as @ref Features::operator==(). +*/ +template::Index)> constexpr bool operator==(T a, Features b) { + return Features(a) == b; +} + +/** @relates Features +@brief Non-equality comparison of a tag and a feature set + +Same as @ref Features::operator!=(). +*/ +template::Index)> constexpr bool operator!=(T a, Features b) { + return Features(a) != b; +} + +/** @relates Features +@brief Whether @p a is a superset of @p b (@f$ a \supseteq b @f$) + +Same as @ref Features::operator>=(). +*/ +template::Index)> constexpr bool operator>=(T a, Features b) { + return Features(a) >= b; +} + +/** @relates Features +@brief Whether @p a is a subset of @p b (@f$ a \subseteq b @f$) + +Same as @ref Features::operator<=(). +*/ +template::Index)> constexpr bool operator<=(T a, Features b) { + return Features(a) <= b; +} + +/** @relates Features +@brief Union of two feature sets + +Same as @ref Features::operator|(). +*/ +template::Index)> constexpr Features operator|(T a, Features b) { + return b | a; +} + +/** @relates Features +@brief Intersection of two feature sets + +Same as @ref Features::operator&(). +*/ +template::Index)> constexpr Features operator&(T a, Features b) { + return b & a; +} + +/** @relates Features +@brief XOR of two feature sets + +Same as @ref Features::operator^(). +*/ +template::Index)> constexpr Features operator^(T a, Features b) { + return b ^ a; +} + +/** @relates Features +@brief Feature set complement + +Same as @ref Features::operator~(). +*/ +template::Index)> constexpr Features operator~(T a) { + return ~Features(a); +} + +/** @debugoperator{Features} */ +CORRADE_UTILITY_EXPORT Utility::Debug& operator<<(Utility::Debug& debug, Features value); + +/** @relates Features +@overload +*/ +template::Index)> inline Utility::Debug& operator<<(Utility::Debug& debug, T value) { + return operator<<(debug, Features{value}); +} + +/** +@brief CPU instruction sets enabled at compile time + +On @ref CORRADE_TARGET_X86 "x86" returns a combination of @ref Sse2, @ref Sse3, +@ref Ssse3, @ref Sse41, @ref Sse42, @ref Avx, @ref Avx2 and @ref Avx512f based +on what all @ref CORRADE_TARGET_SSE2 etc. preprocessor variables are defined. + +On @ref CORRADE_TARGET_ARM "ARM", returns a combination of @ref Neon, +@ref NeonFma and @ref NeonFp16 based on what all @ref CORRADE_TARGET_NEON etc. +preprocessor variables are defined. + +On @ref CORRADE_TARGET_WASM "WebAssembly", returns @ref Simd128 based on +whether the @ref CORRADE_TARGET_SIMD128 preprocessor variable is defined. + +On other platforms or if no known CPU instruction set is enabled, the returned +value is equal to @ref Scalar, which in turn is equivalent to empty (or +default-constructed) @ref Features. +@see @ref DefaultBase +*/ +constexpr Features compiledFeatures() { + return Features{ + #ifdef CORRADE_TARGET_X86 + #ifdef CORRADE_TARGET_SSE2 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSE3 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSSE3 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSE41 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_SSE42 + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX2 + TypeTraits::Index| + #endif + + #elif defined(CORRADE_TARGET_ARM) + #ifdef CORRADE_TARGET_NEON + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FMA + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FP16 + TypeTraits::Index| + #endif + + #elif defined(CORRADE_TARGET_WASM) + #ifdef CORRADE_TARGET_SIMD128 + TypeTraits::Index| + #endif + #endif + 0}; +} + +/** +@brief Detect available CPU instruction sets at runtime + +On @ref CORRADE_TARGET_X86 "x86" and GCC, Clang or MSVC uses the +[CPUID](https://en.wikipedia.org/wiki/CPUID) builtin to check for the +@ref Sse2, @ref Sse3, @ref Ssse3, @ref Sse41, @ref Sse42, @ref Avx, @ref Avx2 +and @ref Avx512f runtime features. @ref Avx needs OS support as well, if it's +not present, no following flags are checked either. On compilers other than +GCC, Clang and MSVC the function is @cpp constexpr @ce and delegates into +@ref compiledFeatures(). + +On @ref CORRADE_TARGET_ARM "ARM", no runtime detection is implemented at the +moment. The function is @cpp constexpr @ce and delegates into +@ref compiledFeatures(). + +On @ref CORRADE_TARGET_WASM "WebAssembly" an attempt to use SIMD instructions +without runtime support results in a WebAssembly compilation error and thus +runtime detection is largely meaningless. While this may change once the +[feature detection proposal](https://github.com/WebAssembly/feature-detection/blob/main/proposals/feature-detection/Overview.md) +is implemented, at the moment the function is @cpp constexpr @ce and delegates +into @ref compiledFeatures(). + +On other platforms or if no known CPU instruction set is detected, the returned +value is equal to @ref Scalar, which in turn is equivalent to empty (or +default-constructed) @ref Features. + +@see @ref DefaultBase +*/ +#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || defined(DOXYGEN_GENERATING_OUTPUT) +CORRADE_UTILITY_EXPORT Features runtimeFeatures(); +#else +constexpr Features runtimeFeatures() { return compiledFeatures(); } +#endif + +} + +} + +#endif diff --git a/src/Corrade/Test/CMakeLists.txt b/src/Corrade/Test/CMakeLists.txt index 733495046..58d69f368 100644 --- a/src/Corrade/Test/CMakeLists.txt +++ b/src/Corrade/Test/CMakeLists.txt @@ -33,6 +33,8 @@ corrade_add_test(MainTest MainTest.cpp # Prefixed with project name to avoid conflicts with TagsTest in Magnum corrade_add_test(CorradeTagsTest TagsTest.cpp) +corrade_add_test(CpuTest CpuTest.cpp) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/configure.h) corrade_add_test(TargetTest TargetTest.cpp) diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp new file mode 100644 index 000000000..3058139d3 --- /dev/null +++ b/src/Corrade/Test/CpuTest.cpp @@ -0,0 +1,461 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include "Corrade/Cpu.h" +#include "Corrade/Containers/StringView.h" +#include "Corrade/TestSuite/Tester.h" +#include "Corrade/Utility/DebugStl.h" /** @todo remove when is gone */ + +namespace Corrade { namespace Test { namespace { + +struct CpuTest: TestSuite::Tester { + explicit CpuTest(); + + void tagNoDefaultConstructor(); + void tagInlineDefinition(); + + void typeTraits(); + + /* Most of the operator tests is inherited from EnumSetTest, just replacing + Feature::Fast with Cpu::Sse2, Feature::Cheap with Cpu::Sse3, + Feature::Tested with Cpu::Ssse3 and Feature::Popular with Cpu::Sse41, + and adjusting numeric values for inverse because there's no fullValue + specified */ + + void featuresConstructScalar(); + void featuresConstruct(); + void featuresOperatorOr(); + void featuresOperatorAnd(); + void featuresOperatorXor(); + void featuresOperatorBoolScalar(); + void featuresOperatorBool(); + void featuresOperatorInverse(); + void featuresCompare(); + + void detectDefault(); + void detect(); + + void tagDispatch(); + + void debug(); + void debugPacked(); +}; + +const struct { + const char* name; + Cpu::Features(*function)(); +} DetectData[]{ + {"compiled", Cpu::compiledFeatures}, + {"runtime", Cpu::runtimeFeatures} +}; + +CpuTest::CpuTest() { + addTests({&CpuTest::tagNoDefaultConstructor, + &CpuTest::tagInlineDefinition, + + &CpuTest::typeTraits, + + &CpuTest::featuresConstructScalar, + &CpuTest::featuresConstruct, + &CpuTest::featuresOperatorOr, + &CpuTest::featuresOperatorAnd, + &CpuTest::featuresOperatorXor, + &CpuTest::featuresOperatorBoolScalar, + &CpuTest::featuresOperatorBool, + &CpuTest::featuresOperatorInverse, + &CpuTest::featuresCompare, + + &CpuTest::detectDefault}); + + addInstancedTests({&CpuTest::detect}, + Containers::arraySize(DetectData)); + + addTests({&CpuTest::tagDispatch, + + &CpuTest::debug, + &CpuTest::debugPacked}); +} + +using namespace Containers::Literals; + +void CpuTest::tagNoDefaultConstructor() { + /* Isn't default constructible to prevent ambiguity when calling + foo({}) if both foo(TagT) and foo(whatever) is available */ + CORRADE_VERIFY(!std::is_default_constructible::value); + #ifdef CORRADE_TARGET_X86 + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + #elif defined(CORRADE_TARGET_ARM) + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + #elif defined(CORRADE_TARGET_WASM) + CORRADE_VERIFY(!std::is_default_constructible::value); + #endif +} + +void CpuTest::tagInlineDefinition() { + /* Just a sanity check that the types match */ + CORRADE_VERIFY(std::is_same::value); + #ifdef CORRADE_TARGET_X86 + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_ARM) + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_WASM) + CORRADE_VERIFY(std::is_same::value); + #endif +} + +void CpuTest::typeTraits() { + CORRADE_VERIFY(!Cpu::TypeTraits::Index); + #ifdef CORRADE_TARGET_X86 + CORRADE_VERIFY(Cpu::TypeTraits::Index); + CORRADE_COMPARE(Cpu::TypeTraits::name(), "Avx2"_s); + #elif defined(CORRADE_TARGET_ARM) + CORRADE_VERIFY(Cpu::TypeTraits::Index); + CORRADE_COMPARE(Cpu::TypeTraits::name(), "NeonFp16"_s); + #elif defined(CORRADE_TARGET_WASM) + CORRADE_VERIFY(Cpu::TypeTraits::Index); + CORRADE_COMPARE(Cpu::TypeTraits::name(), "Simd128"_s); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + +void CpuTest::featuresConstructScalar() { + Cpu::Features noFeatures1; + Cpu::Features noFeatures2 = Cpu::Scalar; + constexpr Cpu::Features cNoFeatures1; + constexpr Cpu::Features cNoFeatures2 = Cpu::Scalar; + CORRADE_COMPARE(std::uint32_t(noFeatures1), 0); + CORRADE_COMPARE(std::uint32_t(noFeatures2), 0); + CORRADE_COMPARE(std::uint32_t(cNoFeatures1), 0); + CORRADE_COMPARE(std::uint32_t(cNoFeatures2), 0); + + CORRADE_VERIFY(std::is_nothrow_constructible::value); + CORRADE_VERIFY(std::is_nothrow_constructible::value); +} + +void CpuTest::featuresConstruct() { + #ifdef CORRADE_TARGET_X86 + Cpu::Features features = Cpu::Sse3; + constexpr Cpu::Features cFeatures = Cpu::Sse3; + CORRADE_COMPARE(std::uint32_t(features), 2); + CORRADE_COMPARE(std::uint32_t(cFeatures), 2); + CORRADE_VERIFY(std::is_nothrow_constructible::value); + #elif defined(CORRADE_TARGET_ARM) + Cpu::Features features = Cpu::Neon; + constexpr Cpu::Features cFeatures = Cpu::Neon; + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + CORRADE_VERIFY(std::is_nothrow_constructible::value); + #elif defined(CORRADE_TARGET_WASM) + Cpu::Features features = Cpu::Simd128; + constexpr Cpu::Features cFeatures = Cpu::Simd128; + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + CORRADE_VERIFY(std::is_nothrow_constructible::value); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + +void CpuTest::featuresOperatorOr() { + #ifdef CORRADE_TARGET_X86 + Cpu::Features features = Cpu::Sse3|Cpu::Sse2; + CORRADE_COMPARE(std::uint32_t(features), 3); + + CORRADE_COMPARE(std::uint32_t(features|Cpu::Ssse3), 7); + CORRADE_COMPARE(std::uint32_t(Cpu::Ssse3|features), 7); + + features |= Cpu::Ssse3; + CORRADE_COMPARE(std::uint32_t(features), 7); + + constexpr Cpu::Features cFeatures = Cpu::Sse3|Cpu::Sse2; + constexpr Cpu::Features cFeatures1 = cFeatures|Cpu::Ssse3; + constexpr Cpu::Features cFeatures2 = Cpu::Ssse3|cFeatures; + CORRADE_COMPARE(std::uint32_t(cFeatures), 3); + CORRADE_COMPARE(std::uint32_t(cFeatures1), 7); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 7); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorAnd() { + #ifdef CORRADE_TARGET_X86 + CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 & Cpu::Sse2), 0); + + Cpu::Features features = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + CORRADE_COMPARE(std::uint32_t(features & Cpu::Sse41), 8); + CORRADE_COMPARE(std::uint32_t(Cpu::Sse41 & features), 8); + + CORRADE_COMPARE(std::uint32_t(features & Cpu::Ssse3), 0); + + Cpu::Features features2 = Cpu::Sse41|Cpu::Sse2|Cpu::Ssse3; + CORRADE_COMPARE(std::uint32_t(features & features2), 9); + + features &= features2; + CORRADE_COMPARE(std::uint32_t(features), 9); + + constexpr Cpu::Features cFeatures = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr Cpu::Features cFeatures1 = cFeatures & Cpu::Sse41; + constexpr Cpu::Features cFeatures2 = Cpu::Sse41 & cFeatures; + CORRADE_COMPARE(std::uint32_t(cFeatures1), 8); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 8); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorXor() { + #ifdef CORRADE_TARGET_X86 + CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 ^ Cpu::Sse3), 0); + CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 ^ Cpu::Sse2), 3); + + Cpu::Features features = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + CORRADE_COMPARE(std::uint32_t(features ^ Cpu::Sse2), 10); + CORRADE_COMPARE(std::uint32_t(Cpu::Sse2 ^ features), 10); + + CORRADE_COMPARE(std::uint32_t(features ^ Cpu::Sse41), 3); + + Cpu::Features features2 = Cpu::Sse41|Cpu::Sse2|Cpu::Ssse3; + CORRADE_COMPARE(std::uint32_t(features ^ features2), 6); + + features ^= features2; + CORRADE_COMPARE(std::uint32_t(features), 6); + + constexpr Cpu::Features cFeatures = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr Cpu::Features cFeatures1 = cFeatures ^ Cpu::Sse2; + constexpr Cpu::Features cFeatures2 = Cpu::Sse2 ^ cFeatures; + CORRADE_COMPARE(std::uint32_t(cFeatures1), 10); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 10); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorBoolScalar() { + CORRADE_COMPARE(!!Cpu::Features{Cpu::Scalar}, false); + + constexpr bool cFeatures = !!Cpu::Features{Cpu::Scalar}; + CORRADE_VERIFY(!cFeatures); +} + +void CpuTest::featuresOperatorBool() { + #ifdef CORRADE_TARGET_X86 + Cpu::Features features = Cpu::Sse3|Cpu::Sse2; + CORRADE_COMPARE(!!(features & Cpu::Sse41), false); + CORRADE_COMPARE(!!(features & Cpu::Sse3), true); + + constexpr Cpu::Features cFeatures = Cpu::Sse3|Cpu::Sse2; + constexpr bool cFeatures1 = !!(cFeatures & Cpu::Sse41); + constexpr bool cFeatures2 = !!(cFeatures & Cpu::Sse3); + CORRADE_VERIFY(!cFeatures1); + CORRADE_VERIFY(cFeatures2); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresOperatorInverse() { + #ifdef CORRADE_TARGET_X86 + CORRADE_COMPARE(std::uint32_t(~Cpu::Scalar), 0xffffffffu); + CORRADE_COMPARE(std::uint32_t(~(Cpu::Sse41|Cpu::Sse3)), 4294967285u); + CORRADE_COMPARE(std::uint32_t(~Cpu::Sse41), 4294967287u); + + constexpr Cpu::Features cFeatures1 = ~Cpu::Scalar; + constexpr Cpu::Features cFeatures2 = ~(Cpu::Sse41|Cpu::Sse3); + CORRADE_COMPARE(std::uint32_t(cFeatures1), 0xffffffffu); + CORRADE_COMPARE(std::uint32_t(cFeatures2), 4294967285u); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::featuresCompare() { + #ifdef CORRADE_TARGET_X86 + Cpu::Features features = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + CORRADE_VERIFY(features == features); + CORRADE_VERIFY(!(features != features)); + CORRADE_VERIFY(Cpu::Sse3 == Cpu::Sse3); + CORRADE_VERIFY(Cpu::Sse3 != Cpu::Sse41); + + CORRADE_VERIFY(Cpu::Scalar <= Cpu::Sse41); + CORRADE_VERIFY(Cpu::Sse41 >= Cpu::Scalar); + CORRADE_VERIFY(Cpu::Sse41 <= Cpu::Sse41); + CORRADE_VERIFY(Cpu::Sse41 >= Cpu::Sse41); + CORRADE_VERIFY(Cpu::Sse41 <= features); + CORRADE_VERIFY(features >= Cpu::Sse41); + CORRADE_VERIFY(features <= features); + CORRADE_VERIFY(features >= features); + + CORRADE_VERIFY(features <= (Cpu::Sse41|Cpu::Sse2|Cpu::Sse3|Cpu::Ssse3)); + CORRADE_VERIFY(!(features >= (Cpu::Sse41|Cpu::Sse2|Cpu::Sse3|Cpu::Ssse3))); + + constexpr Cpu::Features cFeatures = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr bool cFeaturesEqual = cFeatures == cFeatures; + constexpr bool cFeaturesNonEqual = cFeatures != cFeatures; + constexpr bool cFeaturesLessEqual = cFeatures <= cFeatures; + constexpr bool cFeaturesGreaterEqual = cFeatures >= cFeatures; + CORRADE_VERIFY(cFeaturesEqual); + CORRADE_VERIFY(!cFeaturesNonEqual); + CORRADE_VERIFY(cFeaturesLessEqual); + CORRADE_VERIFY(cFeaturesGreaterEqual); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::detectDefault() { + CORRADE_INFO("Detected:" << Debug::packed << Cpu::DefaultBase); + + /* There should be at least something if we have any of the defines + present */ + #if defined(CORRADE_TARGET_SSE2) || defined(CORRADE_TARGET_SSE3) || defined(CORRADE_TARGET_SSSE3) || defined(CORRADE_TARGET_SSE41) || defined(CORRADE_TARGET_SSE42) || defined(CORRADE_TARGET_AVX) || defined(CORRADE_TARGET_AVX2) || defined(CORRADE_TARGET_NEON) || defined(CORRADE_TARGET_SIMD128) + CORRADE_VERIFY(Cpu::Features{Cpu::DefaultBase}); + + /* And nothing if we don't */ + #else + CORRADE_VERIFY(!Cpu::Features{Cpu::DefaultBase}); + #endif +} + +void CpuTest::detect() { + auto&& data = DetectData[testCaseInstanceId()]; + setTestCaseDescription(data.name); + + Cpu::Features features = data.function(); + CORRADE_INFO("Detected:" << Debug::packed << features); + + /* The compile-time feature should be listed among these as well, otherwise + we wouldn't even be able to run the code. */ + CORRADE_VERIFY(features >= Cpu::DefaultBase); + + #ifdef CORRADE_TARGET_X86 + /* Test that for every feature, the subset is present as well */ + if(features & Cpu::Avx512f) CORRADE_VERIFY(features & Cpu::Avx2); + if(features & Cpu::Avx2) CORRADE_VERIFY(features & Cpu::Avx); + if(features & Cpu::Avx) CORRADE_VERIFY(features & Cpu::Sse42); + if(features & Cpu::Sse42) CORRADE_VERIFY(features & Cpu::Sse41); + if(features & Cpu::Sse41) CORRADE_VERIFY(features & Cpu::Ssse3); + if(features & Cpu::Ssse3) CORRADE_VERIFY(features & Cpu::Sse3); + if(features & Cpu::Sse3) CORRADE_VERIFY(features & Cpu::Sse2); + #elif defined(CORRADE_TARGET_ARM) + if(features & Cpu::NeonFp16) CORRADE_VERIFY(features & Cpu::NeonFma); + if(features & Cpu::NeonFma) CORRADE_VERIFY(features & Cpu::Neon); + #else + /* WebAssembly currently has just one feature, so no subset testing applies + on those */ + #endif +} + +#if defined(CORRADE_TARGET_X86) || defined(CORRADE_TARGET_ARM) +const char* dispatch(Cpu::ScalarT) { return "scalar"; } +#ifdef CORRADE_TARGET_X86 +const char* dispatch(Cpu::Sse3T) { return "SSE3"; } +const char* dispatch(Cpu::Avx2T) { return "AVX2"; } +#elif defined(CORRADE_TARGET_ARM) +const char* dispatch(Cpu::NeonT) { return "NEON"; } +const char* dispatch(Cpu::NeonFmaT) { return "NEON FMA"; } +#else +#error +#endif +#endif + +void CpuTest::tagDispatch() { + #ifdef CORRADE_TARGET_X86 + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatch(Cpu::Avx512f), "AVX2"_s); + CORRADE_COMPARE(dispatch(Cpu::Sse42), "SSE3"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatch(Cpu::Sse3), "SSE3"_s); + + /* Anything below gets ... the scalar */ + CORRADE_COMPARE(dispatch(Cpu::Sse2), "scalar"_s); + #elif defined(CORRADE_TARGET_ARM) + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatch(Cpu::NeonFp16), "NEON FMA"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatch(Cpu::Neon), "NEON"_s); + CORRADE_COMPARE(dispatch(Cpu::Scalar), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::debug() { + /* Features{} are equivalent to Scalar */ + #ifdef CORRADE_TARGET_X86 + std::ostringstream out; + Debug{&out} << Cpu::Scalar << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Cpu::Features{}; + CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Ssse3|Cpu::Sse41|Cpu::Avx2 Cpu::Scalar\n"); + #elif defined(CORRADE_TARGET_ARM) + std::ostringstream out; + Debug{&out} << Cpu::Scalar << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Cpu::Features{}; + CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Neon|Cpu::NeonFma|Cpu::NeonFp16 Cpu::Scalar\n"); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +void CpuTest::debugPacked() { + /* Features{} are equivalent to Scalar */ + #ifdef CORRADE_TARGET_X86 + std::ostringstream out; + Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Debug::packed << Cpu::Features{} << Cpu::Avx; + CORRADE_COMPARE(out.str(), "Scalar Ssse3|Sse41|Avx2 Scalar Cpu::Avx\n"); + #elif defined(CORRADE_TARGET_ARM) + std::ostringstream out; + Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Debug::packed << Cpu::Features{} << Cpu::NeonFma; + CORRADE_COMPARE(out.str(), "Scalar Neon|NeonFma|NeonFp16 Scalar Cpu::NeonFma\n"); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +}}} + +CORRADE_TEST_MAIN(Corrade::Test::CpuTest) diff --git a/src/Corrade/Utility/CMakeLists.txt b/src/Corrade/Utility/CMakeLists.txt index 24c6714ab..7951a42da 100644 --- a/src/Corrade/Utility/CMakeLists.txt +++ b/src/Corrade/Utility/CMakeLists.txt @@ -38,6 +38,8 @@ if(CORRADE_WITH_UTILITY) Sha1.cpp System.cpp + ../Cpu.cpp + Implementation/ErrorString.cpp) set(CorradeUtility_GracefulAssert_SRCS diff --git a/src/Corrade/Utility/Memory.h b/src/Corrade/Utility/Memory.h index 6dd1d5a09..6a3712398 100644 --- a/src/Corrade/Utility/Memory.h +++ b/src/Corrade/Utility/Memory.h @@ -105,6 +105,8 @@ loop or a call to @ref std::memset(). constructor on all elements manually using placement new, @ref std::uninitialized_copy() or similar --- see the function docs for an example. + +@see @ref Cpu */ template inline Containers::Array allocateAligned(std::size_t size); @@ -121,7 +123,7 @@ differing behavior for trivial types it's better to explicitly use either the Implemented via @ref allocateAligned(NoInitT, std::size_t) with a loop calling the constructors on the returned allocation in case of non-trivial types. -@see @ref allocateAligned(ValueInitT, std::size_t) +@see @ref allocateAligned(ValueInitT, std::size_t), @ref Cpu */ template Containers::Array allocateAligned(DefaultInitT, std::size_t size); @@ -133,7 +135,7 @@ Same as @ref allocateAligned(std::size_t), just more explicit. Implemented via @ref allocateAligned(NoInitT, std::size_t) with either a @ref std::memset() or a loop calling the constructors on the returned allocation. -@see @ref allocateAligned(DefaultInitT, std::size_t) +@see @ref allocateAligned(DefaultInitT, std::size_t), @ref Cpu */ template Containers::Array allocateAligned(ValueInitT, std::size_t size); @@ -153,7 +155,7 @@ uninitialized memory: @snippet Utility.cpp allocateAligned-NoInit @see @ref allocateAligned(DefaultInitT, std::size_t), - @ref allocateAligned(ValueInitT, std::size_t) + @ref allocateAligned(ValueInitT, std::size_t), @ref Cpu */ template Containers::Array allocateAligned(NoInitT, std::size_t size); From 32219902ccae591d6f5f975ad9e12560cd204949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 9 Jul 2022 19:32:50 +0200 Subject: [PATCH 07/79] Cpu: ability to construct a tag or a Feature from a tag type. Needs this for templated tests. --- doc/snippets/Corrade.cpp | 17 +++++++++++++ src/Corrade/Cpu.h | 48 +++++++++++++++++++++++++++++++++++- src/Corrade/Test/CpuTest.cpp | 46 ++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index a8026d738..07b0bacdb 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -40,6 +40,8 @@ void transform(Cpu::Avx2T, Containers::ArrayView data); /* [Cpu-usage-declare] */ #endif +inline void foo(Cpu::ScalarT) {} + int main() { #ifdef CORRADE_TARGET_X86 { @@ -63,6 +65,21 @@ else transform(Cpu::Scalar, data); /* [Cpu-usage-runtime-manual-dispatch] */ } + +{ +/* [Cpu-tag-from-type] */ +foo(Cpu::Avx2); +foo(Cpu::tag()); +/* [Cpu-tag-from-type] */ +} +{ +/* [Cpu-features-from-type] */ +Cpu::Features a = Cpu::Avx2; +Cpu::Features b = Cpu::features(); +/* [Cpu-features-from-type] */ +static_cast(a); +static_cast(b); +} #endif } diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 5286006f8..6f6d85d47 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -161,6 +161,7 @@ Useful for detecting tag properties at compile time without the need for repeated code such as method overloading, cascaded ifs or template specializations for all tag types. All tag types in the @ref Cpu namespace have this class implemented. +@see @ref tag(), @ref features() */ #ifndef DOXYGEN_GENERATING_OUTPUT template struct TypeTraits; @@ -195,6 +196,7 @@ namespace Implementation { @brief Scalar tag type See the @ref Cpu namespace and the @ref Scalar tag for more information. +@see @ref tag(), @ref features() */ struct ScalarT { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -217,6 +219,7 @@ template<> struct TypeTraits { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Sse2 tag for more information. +@see @ref tag(), @ref features() */ struct Sse2T: ScalarT { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -230,6 +233,7 @@ struct Sse2T: ScalarT { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Sse3 tag for more information. +@see @ref tag(), @ref features() */ struct Sse3T: Sse2T { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -243,6 +247,7 @@ struct Sse3T: Sse2T { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Ssse3 tag for more information. +@see @ref tag(), @ref features() */ struct Ssse3T: Sse3T { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -256,6 +261,7 @@ struct Ssse3T: Sse3T { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Sse41T tag for more information. +@see @ref tag(), @ref features() */ struct Sse41T: Ssse3T { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -269,6 +275,7 @@ struct Sse41T: Ssse3T { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Sse42T tag for more information. +@see @ref tag(), @ref features() */ struct Sse42T: Sse41T { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -282,6 +289,7 @@ struct Sse42T: Sse41T { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Avx tag for more information. +@see @ref tag(), @ref features() */ struct AvxT: Sse42T { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -295,6 +303,7 @@ struct AvxT: Sse42T { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Avx2 tag for more information. +@see @ref tag(), @ref features() */ struct Avx2T: AvxT { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -308,6 +317,7 @@ struct Avx2T: AvxT { Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace and the @ref Avx512f tag for more information. +@see @ref tag(), @ref features() */ struct Avx512fT: Avx2T { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -358,6 +368,7 @@ template<> struct TypeTraits { Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace and the @ref Neon tag for more information. +@see @ref tag(), @ref features() */ struct NeonT: ScalarT { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -371,6 +382,7 @@ struct NeonT: ScalarT { Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace and the @ref NeonFma tag for more information. +@see @ref tag(), @ref features() */ struct NeonFmaT: NeonT { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -384,6 +396,7 @@ struct NeonFmaT: NeonT { Available only on @ref CORRADE_TARGET_ARM "ARM". See the @ref Cpu namespace and the @ref NeonFp16 tag for more information. +@see @ref tag(), @ref features() */ struct NeonFp16T: NeonFmaT { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -414,6 +427,7 @@ template<> struct TypeTraits { Available only on @ref CORRADE_TARGET_WASM "WebAssembly". See the @ref Cpu namespace and the @ref Simd128 tag for more information. +@see @ref tag(), @ref features() */ struct Simd128T: ScalarT { #ifndef DOXYGEN_GENERATING_OUTPUT @@ -654,6 +668,18 @@ capable of detecting the available CPU feature set at runtime. */ constexpr DefaultBaseT DefaultBase{Implementation::Init}; +/** +@brief Tag for a tag type + +Returns a tag corresponding to tag type @p T. The following two expressions are +equivalent: + +@snippet Corrade.cpp Cpu-tag-from-type + +@see @ref features() +*/ +template constexpr T tag() { return T{Implementation::Init}; } + /** @brief Feature set @@ -673,7 +699,11 @@ class Features { */ constexpr explicit Features() noexcept: _data{} {} - /** @brief Construct from a tag */ + /** + * @brief Construct from a tag + * + * @see @ref features() + */ template::Index)> constexpr /*implicit*/ Features(T) noexcept: _data{TypeTraits::Index} {} /** @brief Equality comparison */ @@ -759,6 +789,7 @@ class Features { constexpr explicit operator unsigned int() const { return _data; } private: + template friend constexpr Features features(); friend constexpr Features compiledFeatures(); #if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) /* MSVC demands the export macro to be here as well */ @@ -770,6 +801,21 @@ class Features { unsigned int _data; }; +/** +@brief Feature set for a tag type + +Returns @ref Features with a tag corresponding to tag type @p T, avoiding a +need to form the tag value in order to pass it to @ref Features::Features(T). +The following two expressions are equivalent: + +@snippet Corrade.cpp Cpu-features-from-type + +@see @ref tag() +*/ +template constexpr Features features() { + return Features{TypeTraits::Index}; +} + /** @relates Features @brief Equality comparison of a tag and a feature set diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index 3058139d3..22fad3035 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -38,6 +38,7 @@ struct CpuTest: TestSuite::Tester { void tagNoDefaultConstructor(); void tagInlineDefinition(); + void tagConstructTemplate(); void typeTraits(); @@ -49,6 +50,7 @@ struct CpuTest: TestSuite::Tester { void featuresConstructScalar(); void featuresConstruct(); + void featuresConstructTemplate(); void featuresOperatorOr(); void featuresOperatorAnd(); void featuresOperatorXor(); @@ -77,11 +79,13 @@ const struct { CpuTest::CpuTest() { addTests({&CpuTest::tagNoDefaultConstructor, &CpuTest::tagInlineDefinition, + &CpuTest::tagConstructTemplate, &CpuTest::typeTraits, &CpuTest::featuresConstructScalar, &CpuTest::featuresConstruct, + &CpuTest::featuresConstructTemplate, &CpuTest::featuresOperatorOr, &CpuTest::featuresOperatorAnd, &CpuTest::featuresOperatorXor, @@ -146,6 +150,27 @@ void CpuTest::tagInlineDefinition() { #endif } +void CpuTest::tagConstructTemplate() { + #ifdef CORRADE_TARGET_X86 + auto tag = Cpu::tag(); + constexpr auto cTag = Cpu::tag(); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_ARM) + auto tag = Cpu::tag(); + constexpr auto cTag = Cpu::tag(); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #elif defined(CORRADE_TARGET_WASM) + auto tag = Cpu::tag(); + constexpr auto cTag = Cpu::tag(); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + void CpuTest::typeTraits() { CORRADE_VERIFY(!Cpu::TypeTraits::Index); #ifdef CORRADE_TARGET_X86 @@ -200,6 +225,27 @@ void CpuTest::featuresConstruct() { #endif } +void CpuTest::featuresConstructTemplate() { + #ifdef CORRADE_TARGET_X86 + auto features = Cpu::features(); + constexpr auto cFeatures = Cpu::features(); + CORRADE_COMPARE(std::uint32_t(features), 2); + CORRADE_COMPARE(std::uint32_t(cFeatures), 2); + #elif defined(CORRADE_TARGET_ARM) + auto features = Cpu::features(); + constexpr auto cFeatures = Cpu::features(); + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + #elif defined(CORRADE_TARGET_WASM) + auto features = Cpu::features(); + constexpr auto cFeatures = Cpu::features(); + CORRADE_COMPARE(std::uint32_t(features), 1); + CORRADE_COMPARE(std::uint32_t(cFeatures), 1); + #else + CORRADE_SKIP("No Cpu tags available on this platform"); + #endif +} + void CpuTest::featuresOperatorOr() { #ifdef CORRADE_TARGET_X86 Cpu::Features features = Cpu::Sse3|Cpu::Sse2; From 0711cd3ce05e13e6758a6d27b51532dfdcd3de3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 9 Jul 2022 19:34:13 +0200 Subject: [PATCH 08/79] Cpu: add x86 CORRADE_ENABLE_* function target attribute macros. Needed for GCC and Clang to avoid having to split the implementations across a multitude of tiny files, each having a different -m compiler option set. Wow damn this was painful. Because GCC 4.8 is crap, it took way longer than expected, so it's just the x86 variants right now. ARM and WebAssembly come next, but those aren't really any less painful. --- doc/snippets/Corrade.cpp | 16 ++ src/Corrade/Corrade.h | 24 +- src/Corrade/Cpu.h | 367 +++++++++++++++++++++++++- src/Corrade/Test/CpuTest.cpp | 181 +++++++++++++ src/Corrade/Utility/CMakeLists.txt | 13 + src/Corrade/Utility/IntrinsicsAvx.h | 89 +++++++ src/Corrade/Utility/IntrinsicsSse2.h | 57 ++++ src/Corrade/Utility/IntrinsicsSse3.h | 61 +++++ src/Corrade/Utility/IntrinsicsSse4.h | 78 ++++++ src/Corrade/Utility/IntrinsicsSsse3.h | 61 +++++ 10 files changed, 930 insertions(+), 17 deletions(-) create mode 100644 src/Corrade/Utility/IntrinsicsAvx.h create mode 100644 src/Corrade/Utility/IntrinsicsSse2.h create mode 100644 src/Corrade/Utility/IntrinsicsSse3.h create mode 100644 src/Corrade/Utility/IntrinsicsSse4.h create mode 100644 src/Corrade/Utility/IntrinsicsSsse3.h diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index 07b0bacdb..98cbf0ff7 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -38,6 +38,22 @@ void transform(Cpu::ScalarT, Containers::ArrayView data); void transform(Cpu::Sse42T, Containers::ArrayView data); void transform(Cpu::Avx2T, Containers::ArrayView data); /* [Cpu-usage-declare] */ + +/* [Cpu-usage-target-attributes] */ +void transform(Cpu::ScalarT, Containers::ArrayView data) { + DOXYGEN_ELLIPSIS(static_cast(data);) +} +#ifdef CORRADE_ENABLE_SSE42 +CORRADE_ENABLE_SSE42 void transform(Cpu::Sse42T, Containers::ArrayView data) { + DOXYGEN_ELLIPSIS(static_cast(data);) +} +#endif +#ifdef CORRADE_ENABLE_AVX2 +CORRADE_ENABLE_AVX2 void transform(Cpu::Avx2T, Containers::ArrayView data) { + DOXYGEN_ELLIPSIS(static_cast(data);) +} +#endif +/* [Cpu-usage-target-attributes] */ #endif inline void foo(Cpu::ScalarT) {} diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 04434213b..e9c148735 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -514,7 +514,8 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if [Streaming SIMD Extensions 2](https://en.wikipedia.org/wiki/SSE2) are enabled at compile time (`-msse2` or higher on GCC/Clang, `/arch:SSE2` or higher on MSVC). All x86-64 targets support SSE2. Implied by @ref CORRADE_TARGET_SSE3. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse2} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse2}, + @ref CORRADE_ENABLE_SSE2 */ #define CORRADE_TARGET_SSE2 #undef CORRADE_TARGET_SSE2 @@ -528,7 +529,8 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if at compile time (on GCC/Clang it's `-msse3` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE2, implied by @ref CORRADE_TARGET_SSSE3. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse3} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse3}, + @ref CORRADE_ENABLE_SSE3 */ #define CORRADE_TARGET_SSE3 #undef CORRADE_TARGET_SSE3 @@ -547,7 +549,8 @@ Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wik but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref CORRADE_TARGET_SSE41 to detect those. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Ssse3} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Ssse3}, + @ref CORRADE_ENABLE_SSSE3 */ #define CORRADE_TARGET_SSSE3 #undef CORRADE_TARGET_SSSE3 @@ -566,7 +569,8 @@ Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wik but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref CORRADE_TARGET_SSE41 to detect those. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse41} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse41}, + @ref CORRADE_ENABLE_SSE41 */ #define CORRADE_TARGET_SSE41 #undef CORRADE_TARGET_SSE41 @@ -580,7 +584,8 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if are enabled at compile time (on GCC/Clang it's `-msse4.2` and higher, MSVC doesn't have a direct option and it's only implied by `/arch:AVX`). Superset of @ref CORRADE_TARGET_SSE41, implied by @ref CORRADE_TARGET_AVX. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse42} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Sse42}, + @ref CORRADE_ENABLE_SSE42 */ #define CORRADE_TARGET_SSE42 #undef CORRADE_TARGET_SSE42 @@ -623,7 +628,8 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if are enabled at compile time (`-mavx` and higher on GCC/Clang, `/arch:AVX` on MSVC). Superset of @ref CORRADE_TARGET_SSE42, implied by @ref CORRADE_TARGET_AVX2. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx}, + @ref CORRADE_ENABLE_AVX */ #define CORRADE_TARGET_AVX #undef CORRADE_TARGET_AVX @@ -667,7 +673,8 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if are enabled at compile time (`-mavx2` and higher on GCC/Clang, `/arch:AVX2` on MSVC). Superset of @ref CORRADE_TARGET_AVX, implied by @ref CORRADE_TARGET_AVX512F. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx2} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx2}, + @ref CORRADE_ENABLE_AVX2 */ #define CORRADE_TARGET_AVX2 #undef CORRADE_TARGET_AVX2 @@ -679,7 +686,8 @@ MSVC). Superset of @ref CORRADE_TARGET_AVX, implied by Defined on @ref CORRADE_TARGET_X86 "x86" if [AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation instructions are enabled at compile time (`-mavx512f` and higher on GCC/Clang, `/arch:AVX512` on MSVC). Superset of @ref CORRADE_TARGET_AVX2. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx512f} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Avx512f}, + @ref CORRADE_ENABLE_AVX512F */ #define CORRADE_TARGET_AVX512F #undef CORRADE_TARGET_AVX512F diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 6f6d85d47..c82c84ca8 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -27,7 +27,7 @@ */ /** @file - * @brief Namespace @ref Corrade::Cpu + * @brief Namespace @ref Corrade::Cpu and related macros * @m_since_latest */ @@ -151,6 +151,46 @@ returned @ref Cpu::Features against particular tags to decide which variant to use: @snippet Corrade.cpp Cpu-usage-runtime-manual-dispatch + +@subsection Cpu-usage-target-attributes Enabling instruction sets for particular functions + +On GCC and Clang, a machine target has to be enabled in order to use a +particular CPU instruction set or its intrinsics. While it's possible to do +that for the whole compilation unit by passing for example `-mavx2` to the +compiler, it would force you to create dedicated files for every architecture +variant you want to support. Instead, it's possible to equip particular +functions with *target attributes* defined by @ref CORRADE_ENABLE_SSE2 and +related macros, which then makes a particular instruction set enabled for given +function. + +In contrast, MSVC doesn't restrict intrinsics usage in any way, so you can +freely call e.g. AVX2 intrinsics even if the whole file is compiled with just +SSE2 enabled. The @ref CORRADE_ENABLE_SSE2 and related macros are thus defined +to be empty on this compiler. + +@m_class{m-note m-warning} + +@par + On the other hand, on MSVC, using just the baseline target on the file + level means the compiler will not be able to use any advanced instructions + apart from what you call explicitly via intrinsics. You can try extracting + all AVX+ variants into a dedicated file with `/arch:AVX` enabled and see + if it makes any performance difference. + +For developer convenience, the @ref CORRADE_ENABLE_SSE2 etc. macros are defined +only on matching architectures, and generally only if the compiler itself has +given feature set implemented and usable. Which means you can easily use them +to @cpp #ifdef @ce your variants to be compiled only where it makes sense, or +even guard intrinsics includes with them to avoid including potentially heavy +headers you won't use anyway. In comparison, using the @ref CORRADE_TARGET_SSE2 +etc. macros would only make the variant available if the whole compilation unit +has a corresponding `-m` or `/arch:` option passed to the compiler. + +Definitions of the above functions would then look like below with the target +attributes enabled. The scalar variant has no target-specific annotation and +will be present as a fallback in all cases. + +@snippet Corrade.cpp Cpu-usage-target-attributes */ namespace Cpu { @@ -462,7 +502,7 @@ constexpr ScalarT Scalar{Implementation::Init}; only on @ref CORRADE_TARGET_X86 "x86", supported by all 64-bit x86 processors and is present on majority of contemporary 32-bit x86 processors as well. Superset of @ref Scalar, implied by @ref Sse3. -@see @ref CORRADE_TARGET_SSE2 +@see @ref CORRADE_TARGET_SSE2, @ref CORRADE_ENABLE_SSE2 */ constexpr Sse2T Sse2{Implementation::Init}; @@ -472,7 +512,7 @@ constexpr Sse2T Sse2{Implementation::Init}; [Streaming SIMD Extensions 3](https://en.wikipedia.org/wiki/SSE3). Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse2, implied by @ref Ssse3. -@see @ref CORRADE_TARGET_SSE3 +@see @ref CORRADE_TARGET_SSE3, @ref CORRADE_ENABLE_SSE3 */ constexpr Sse3T Sse3{Implementation::Init}; @@ -486,7 +526,7 @@ by @ref Sse41. Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wiki/SSE4#SSE4a) but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref Sse41 to handle those. -@see @ref CORRADE_TARGET_SSSE3 +@see @ref CORRADE_TARGET_SSSE3, @ref CORRADE_ENABLE_SSSE3 */ constexpr Ssse3T Ssse3{Implementation::Init}; @@ -500,7 +540,7 @@ implied by @ref Sse42. Note that certain older AMD processors have [SSE4a](https://en.wikipedia.org/wiki/SSE4#SSE4a) but neither SSSE3 nor SSE4.1. Both can be however treated as a subset of SSE4.1 to a large extent, and it's recommended to use @ref Sse41 to handle those. -@see @ref CORRADE_TARGET_SSE41 +@see @ref CORRADE_TARGET_SSE41, @ref CORRADE_ENABLE_SSE41 */ constexpr Sse41T Sse41{Implementation::Init}; @@ -510,7 +550,7 @@ constexpr Sse41T Sse41{Implementation::Init}; [Streaming SIMD Extensions 4.2](https://en.wikipedia.org/wiki/SSE4#SSE4.2). Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse41, implied by @ref Avx. -@see @ref CORRADE_TARGET_SSE42 +@see @ref CORRADE_TARGET_SSE42, @ref CORRADE_ENABLE_SSE42 */ constexpr Sse42T Sse42{Implementation::Init}; @@ -520,7 +560,7 @@ constexpr Sse42T Sse42{Implementation::Init}; [Advanced Vector Extensions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions). Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Sse42, implied by @ref Avx2. -@see @ref CORRADE_TARGET_AVX +@see @ref CORRADE_TARGET_AVX, @ref CORRADE_ENABLE_AVX */ constexpr AvxT Avx{Implementation::Init}; @@ -530,7 +570,7 @@ constexpr AvxT Avx{Implementation::Init}; [Advanced Vector Extensions 2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2). Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Avx, implied by @ref Avx512f. -@see @ref CORRADE_TARGET_AVX2 +@see @ref CORRADE_TARGET_AVX2, @ref CORRADE_ENABLE_AVX2 */ constexpr Avx2T Avx2{Implementation::Init}; @@ -539,7 +579,7 @@ constexpr Avx2T Avx2{Implementation::Init}; [AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation. Available only on @ref CORRADE_TARGET_X86 "x86". Superset of @ref Avx2. -@see @ref CORRADE_TARGET_AVX512F +@see @ref CORRADE_TARGET_AVX512F, @ref CORRADE_ENABLE_AVX512F */ constexpr Avx512fT Avx512f{Implementation::Init}; #endif @@ -995,6 +1035,315 @@ CORRADE_UTILITY_EXPORT Features runtimeFeatures(); constexpr Features runtimeFeatures() { return compiledFeatures(); } #endif +#if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief Enable SSE2 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("sse2"))) @ce, allowing use of +[SSE2](https://en.wikipedia.org/wiki/SSE2) and earlier SSE instructions inside +a function annotated with this macro without having to specify `-msse2` for the +whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Not defined on other compilers or +architectures. + +As a special case, if @ref CORRADE_TARGET_SSE2 is present (meaning SSE2 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Implied by @ref CORRADE_ENABLE_SSE3. See @ref Cpu-usage-target-attributes for +more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSse2.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Sse2} +*/ +#if defined(CORRADE_TARGET_SSE2) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE2 +#elif defined(CORRADE_TARGET_GCC) +#define CORRADE_ENABLE_SSE2 __attribute__((__target__("sse2"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE2 +#endif + +/** +@brief Enable SSE3 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("sse3"))) @ce, allowing use of +[SSE3](https://en.wikipedia.org/wiki/SSE3) and earlier SSE intrinsics inside a +function annotated with this macro without having to specify `-msse3` for the +whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Not defined on other compilers or +architectures. + +As a special case, if @ref CORRADE_TARGET_SSE3 is present (meaning SSE3 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE2, implied by @ref CORRADE_ENABLE_SSSE3. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSse3.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Sse3} +*/ +#if defined(CORRADE_TARGET_SSE3) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE3 +#elif defined(CORRADE_TARGET_GCC) +/* The -msse3 option implies -msse2 on both GCC and Clang, so no need to + specify those as well (verified with `echo | gcc -dM -E - -msse3`) */ +#define CORRADE_ENABLE_SSE3 __attribute__((__target__("sse3"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE3 +#endif + +/** +@brief Enable SSSE3 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("ssse3"))) @ce, allowing use of +[SSSE3](https://en.wikipedia.org/wiki/SSSE3) and earlier SSE instructions +inside a function annotated with this macro without having to specify `-mssse3` +for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler +doesn't restrict use of intrinsics in any way. Not defined on other compilers +or architectures. + +As a special case, if @ref CORRADE_TARGET_SSSE3 is present (meaning SSSE3 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE3, implied by @ref CORRADE_ENABLE_SSE41. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSsse3.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Ssse3} +*/ +#if defined(CORRADE_TARGET_SSSE3) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSSE3 +#elif defined(CORRADE_TARGET_GCC) +/* The -mssse3 option implies -msse2 -msse3 on both GCC and Clang, so no need + to specify those as well (verified with `echo | gcc -dM -E - -mssse3`) */ +#define CORRADE_ENABLE_SSSE3 __attribute__((__target__("ssse3"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSSE3 +#endif + +/** +@brief Enable SSE4.1 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("sse4.1"))) @ce, allowing use of +[SSE4.1](https://en.wikipedia.org/wiki/SSE4#SSE4.1) and earlier SSE +instructions inside a function annotated with this macro without having to +specify `-msse4.1` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Not +defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_SSE41 is present (meaning SSE4.1 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSSE3, implied by @ref CORRADE_ENABLE_SSE42. +See @ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-danger} + +@par + Unless @ref CORRADE_TARGET_SSE41 is present, this macro is not defined on + GCC 4.8, as SSE4.1 intrinsics only work if `-msse4.2` is specified as well + due to both SSE4.1 and 4.2 intrinsics living in the same header. You can + only use @ref CORRADE_ENABLE_SSE42 in this case. + +@see @relativeref{Corrade,Cpu::Sse41} +*/ +#if defined(CORRADE_TARGET_SSE41) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE41 +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) +/* The -msse4.1 option implies -msse2 -msse3 -mssse3 on both GCC and Clang, so + no need to specify those as well (verified with + `echo | gcc -dM -E - -msse4.1`) */ +#define CORRADE_ENABLE_SSE41 __attribute__((__target__("sse4.1"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE41 +#endif + +/** +@brief Enable SSE4.2 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("sse4.2"))) @ce, allowing use of +[SSE4.2](https://en.wikipedia.org/wiki/SSE4#SSE4.2) and earlier SSE +instructions inside a function annotated with this macro without having to +specify `-msse4.2` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Not +defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_SSE42 is defined (meaning SSE4.2 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE41, implied by @ref CORRADE_ENABLE_AVX. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSse4.h instead of + @cpp #include @ce and @cpp #include @ce to be + able to access the intrinsics on this compiler. + +@see @relativeref{Corrade,Cpu::Sse42} +*/ +#if defined(CORRADE_TARGET_SSE42) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SSE42 +#elif defined(CORRADE_TARGET_GCC) +/* The -msse4.2 option implies -msse2 -msse3 -mssse3 -msse4.1 on both GCC and + Clang, so no need to specify those as well (verified with + `echo | gcc -dM -E - -msse4.2`) */ +#define CORRADE_ENABLE_SSE42 __attribute__((__target__("sse4.2"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_SSE42 +#endif + +/** +@brief Enable AVX for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("avx"))) @ce, allowing use of +[AVX](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) and all earlier +SSE instructions inside a function annotated with this macro without having to +specify `-mavx` for the whole compilation unit. On x86 MSVC expands to nothing, +as the compiler doesn't restrict use of intrinsics in any way. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX is present (meaning AVX is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_SSE42, implied by @ref CORRADE_ENABLE_AVX2. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Avx} +*/ +#if defined(CORRADE_TARGET_AVX) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX +#elif defined(CORRADE_TARGET_GCC) +/* The -mavx option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 on both GCC + and Clang, so no need to specify those as well (verified with + `echo | gcc -dM -E - -mavx`) */ +#define CORRADE_ENABLE_AVX __attribute__((__target__("avx"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_AVX +#endif + +/** +@brief Enable AVX2 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("avx2"))) @ce, allowing use of +[AVX2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Advanced_Vector_Extensions_2), +FMA, F16C, AVX and all earlier SSE instructions inside a function annotated +with this macro without having to specify `-mavx2` for the whole compilation +unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of +intrinsics in any way. Not defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX2 is present (meaning AVX2 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Superset of @ref CORRADE_ENABLE_AVX, implied by @ref CORRADE_ENABLE_AVX512F. +See @ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Avx2} +*/ +#if defined(CORRADE_TARGET_AVX2) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX2 +#elif defined(CORRADE_TARGET_GCC) +/* The -mavx2 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on + both GCC and Clang, so no need to specify those as well (verified with + `echo | gcc -dM -E - -mavx2`) */ +#define CORRADE_ENABLE_AVX2 __attribute__((__target__("avx2"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_AVX2 +#endif + +/** +@brief Enable AVX-512 Foundation for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC 4.9+ and Clang expands to +@cpp __attribute__((__target__("avx512f"))) @ce, allowing use of +[AVX-512](https://en.wikipedia.org/wiki/AVX-512) Foundation and all earlier AVX +and SSE instructions inside a function annotated with this macro without having +to specify `-mavx512f` for the whole compilation unit. On x86 MSVC 2017 15.3+ +expands to nothing, as the compiler doesn't restrict use of intrinsics in any +way. Not defined on other compilers, earlier compiler versions without AVX-512 +support or other architectures. + +As a special case, if @ref CORRADE_TARGET_AVX512F is present (meaning AVX-512 +Foundation is enabled for the whole compilation unit), this macro is defined as +empty on all compilers. + +Superset of @ref CORRADE_ENABLE_AVX2. See @ref Cpu-usage-target-attributes for +more information and usage example. +@see @relativeref{Corrade,Cpu::Avx512f} +*/ +#if defined(CORRADE_TARGET_AVX512F) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX512F +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) +/* The -mavx512 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx + -mavx2 on both GCC and Clang, so no need to specify those as well (verified + with `echo | gcc -dM -E - -mavx512f`) */ +#define CORRADE_ENABLE_AVX512F __attribute__((__target__("avx512f"))) +#elif defined(CORRADE_TARGET_MSVC) && _MSC_VER >= 1911 +#define CORRADE_ENABLE_AVX512F +#endif +#endif + } } diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index 22fad3035..b53645633 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -31,6 +31,22 @@ #include "Corrade/TestSuite/Tester.h" #include "Corrade/Utility/DebugStl.h" /** @todo remove when is gone */ +#ifdef CORRADE_ENABLE_SSE2 +#include "Corrade/Utility/IntrinsicsSse2.h" +#endif +#ifdef CORRADE_ENABLE_SSE3 +#include "Corrade/Utility/IntrinsicsSse3.h" +#endif +#ifdef CORRADE_ENABLE_SSSE3 +#include "Corrade/Utility/IntrinsicsSsse3.h" +#endif +#if defined(CORRADE_ENABLE_SSE41) || defined(CORRADE_ENABLE_SSE42) +#include "Corrade/Utility/IntrinsicsSse4.h" +#endif +#if defined(CORRADE_ENABLE_AVX) || defined(CORRADE_ENABLE_AVX_F16C) || defined(CORRADE_ENABLE_AVX_FMA) || defined(CORRADE_ENABLE_AVX2) || defined(CORRADE_ENABLE_AVX512F) +#include "Corrade/Utility/IntrinsicsAvx.h" +#endif + namespace Corrade { namespace Test { namespace { struct CpuTest: TestSuite::Tester { @@ -63,6 +79,7 @@ struct CpuTest: TestSuite::Tester { void detect(); void tagDispatch(); + template void enableMacros(); void debug(); void debugPacked(); @@ -100,6 +117,16 @@ CpuTest::CpuTest() { Containers::arraySize(DetectData)); addTests({&CpuTest::tagDispatch, + #ifdef CORRADE_TARGET_X86 + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + #endif &CpuTest::debug, &CpuTest::debugPacked}); @@ -472,6 +499,160 @@ void CpuTest::tagDispatch() { #endif } +/* Not using an argument here since we *don't* want the overload delegating + in this case -- it would hide errors when a certain instruction set doesn't + have a corresponding overload, as it'd fall back to a parent one. I'm also + defining a catch-all implementation with CORRADE_SKIP() instead of having an + #ifdef CORRADE_ENABLE_* around every variant in addTests(), because this way + it's clearly visible in the test output if any CORRADE_ENABLE_* macro isn't + defined for whatever reason. */ +template int callInstructionFor() { + CORRADE_SKIP("No CORRADE_ENABLE_* macro for" << Cpu::features() << "on this compiler"); +} +/* The goal here is to use instructions that would make the compilation fail + on GCC and default flags (i.e., no -march=native etc.) if the + CORRADE_ENABLE_* macro is removed. While this is quite a lot of code, it's a + good overview of how all the instructions look like... and it also uncovers + a MASSIVE amount of platform-specific warts and compiler bugs that the API + should take care of. + + All these are also marked with CORRADE_NEVER_INLINE to make it easier to see + into what code they get actually compiled. Except for the catch-all variant, + which isn't interesting for disassembly. */ +#ifdef CORRADE_ENABLE_SSE2 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE2 int callInstructionFor() { + __m128i a = _mm_set_epi32(0x80808080, 0, 0x80808080, 0); + + /* All instructions SSE2 */ + + int mask = _mm_movemask_epi8(a); + CORRADE_COMPARE(mask, 0xf0f0); /* 0b1111000011110000 */ + return mask; +} +#endif +#ifdef CORRADE_ENABLE_SSE3 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE3 int callInstructionFor() { + const std::uint32_t a[]{0, 10, 20, 30, 40}; + + /* SSE3 */ + union { + __m128i v; + int s[4]; + } b; + b.v = _mm_lddqu_si128(reinterpret_cast(a + 1)); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_SSSE3 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSSE3 int callInstructionFor() { + __m128i a = _mm_set_epi32(-10, 20, -30, 40); + + /* SSSE3 */ + union { + __m128i v; + int s[4]; + } b; + b.v = _mm_abs_epi32(a); + + CORRADE_COMPARE(b.s[3], 10); + CORRADE_COMPARE(b.s[2], 20); + CORRADE_COMPARE(b.s[1], 30); + CORRADE_COMPARE(b.s[0], 40); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_SSE41 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE41 int callInstructionFor() { + __m128 a = _mm_set_ps(5.47f, 2.23f, 7.62f, 0.5f); + + /* SSE4.1 */ + union { + __m128 v; + float s[4]; + } b; + b.v = _mm_ceil_ps(a); + + CORRADE_COMPARE(b.s[3], 6.0f); + CORRADE_COMPARE(b.s[2], 3.0f); + CORRADE_COMPARE(b.s[1], 8.0f); + CORRADE_COMPARE(b.s[0], 1.0f); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_SSE42 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE42 int callInstructionFor() { + __m128i a = _mm_set_epi64x(50, 60); + __m128i b = _mm_set_epi64x(60, 50); + + /* SSE4.2 */ + union { + __m128i v; + std::int64_t s[2]; + } c; + c.v = _mm_cmpgt_epi64(a, b); + + CORRADE_COMPARE(c.s[0], -1); + CORRADE_COMPARE(c.s[1], 0); + return c.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_AVX +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX int callInstructionFor() { + __m256d a = _mm256_set_pd(5.47, 2.23, 7.62, 0.5); + + /* All instructions AVX */ + + union { + __m256d v; + double s[4]; + } b; + b.v = _mm256_ceil_pd(a); + + CORRADE_COMPARE(b.s[3], 6.0); + CORRADE_COMPARE(b.s[2], 3.0); + CORRADE_COMPARE(b.s[1], 8.0); + CORRADE_COMPARE(b.s[0], 1.0); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_AVX2 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX2 int callInstructionFor() { + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* Like callInstructionFor(), but expanded to AVX2 */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00);/* 0b11111111000000001111111100000000 */ + return mask; +} +#endif +#ifdef CORRADE_ENABLE_AVX512F +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX512F int callInstructionFor() { + __m128 a = _mm_set1_ps(5.47f); + + /* AVX512 */ + int ceil = _mm_cvt_roundss_si32(a, _MM_FROUND_TO_POS_INF|_MM_FROUND_NO_EXC); + + CORRADE_COMPARE(ceil, 6); + return ceil; +} +#endif + +template void CpuTest::enableMacros() { + setTestCaseTemplateName(Cpu::TypeTraits::name()); + + if(!(Cpu::runtimeFeatures() & Cpu::features())) + CORRADE_SKIP("CPU feature not supported"); + + CORRADE_VERIFY(true); /* to capture correct function name */ + CORRADE_VERIFY(callInstructionFor()); +} + void CpuTest::debug() { /* Features{} are equivalent to Scalar */ #ifdef CORRADE_TARGET_X86 diff --git a/src/Corrade/Utility/CMakeLists.txt b/src/Corrade/Utility/CMakeLists.txt index 7951a42da..55d74e0c0 100644 --- a/src/Corrade/Utility/CMakeLists.txt +++ b/src/Corrade/Utility/CMakeLists.txt @@ -75,6 +75,19 @@ if(CORRADE_WITH_UTILITY) Format.h FormatStl.h FormatStlStringView.h + + # Because various platforms can have multi-architecture binaries + # (macOS, iOS), the CORRADE_TARGET_X86 variable isn't exposed to CMake. + # And because all desktop and mobile platfoms have an x86 variant + # including iOS and Android, and Emscripten has x86 intrinsics + # compatibility (!!), we install the x86-specific intrinsics wrappers + # always. + IntrinsicsSse2.h + IntrinsicsSse3.h + IntrinsicsSsse3.h + IntrinsicsSse4.h + IntrinsicsAvx.h + Json.h JsonWriter.h Macros.h diff --git a/src/Corrade/Utility/IntrinsicsAvx.h b/src/Corrade/Utility/IntrinsicsAvx.h new file mode 100644 index 000000000..0f9203046 --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsAvx.h @@ -0,0 +1,89 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 AVX and AVX2 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_AVX or @ref CORRADE_ENABLE_AVX2 +function attributes instead of having to specify `-mavx` or `-mavx2` for the +whole compilation unit. + +As AVX-512 is supported only since GCC 4.9, which doesn't need this workaround, +it's not handled here. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSse3.h, @ref IntrinsicsSsse3.h, + @ref IntrinsicsSse4.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If I wouldn't need the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE headers explicitly before, in correct + order */ +#include "Corrade/Utility/IntrinsicsSse4.h" + +/* Include just AVX instructions first. If we would add target("avx2") and + __AVX2__ here as well, it would cause the AVX instruction to be usable only + if target("avx2") is specified as well, which is not a good thing for + CORRADE_ENABLE_AVX, where the code should *only* use AVX at most. The AVX2 + instructions are included below by defining target("avx2") and directly + pulling in , and only doing that on GCC 4.8, as all other + compilers have everything already included from the top-level . + + I wonder what impact this has on optimization, but I don't care that much as + GCC 4.8 is mainly for backwards compatibility testing now, and not serious + performance use. In any case, one can always compile everything with + -march=native to get rid of such potential suboptimal optimization. */ +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("avx") +#pragma push_macro("__AVX__") +#ifndef __AVX__ +#define __AVX__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__AVX__") +#pragma GCC pop_options +#endif + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("avx2") +#pragma push_macro("__AVX2__") +#ifndef __AVX2__ +#define __AVX2__ +#endif +#include +#pragma pop_macro("__AVX2__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSse2.h b/src/Corrade/Utility/IntrinsicsSse2.h new file mode 100644 index 000000000..e667cc71a --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSse2.h @@ -0,0 +1,57 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSE2 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_SSE2 function attribute instead of +having to specify `-msse2` for the whole compilation unit. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse3.h, @ref IntrinsicsSsse3.h, @ref IntrinsicsSse4.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If it wouldn't be for the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("sse2") +#pragma push_macro("__SSE2__") +#ifndef __SSE2__ +#define __SSE2__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSE2__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSse3.h b/src/Corrade/Utility/IntrinsicsSse3.h new file mode 100644 index 000000000..8e04be7ec --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSse3.h @@ -0,0 +1,61 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSE3 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_SSE3 function attribute instead of +having to specify `-msse3` for the whole compilation unit. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSsse3.h, @ref IntrinsicsSse4.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If I wouldn't need the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE2 header explicitly before, in correct + order */ +#include "Corrade/Utility/IntrinsicsSse2.h" + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("sse3") +#pragma push_macro("__SSE3__") +#ifndef __SSE3__ +#define __SSE3__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSE3__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSse4.h b/src/Corrade/Utility/IntrinsicsSse4.h new file mode 100644 index 000000000..92d4b9bc7 --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSse4.h @@ -0,0 +1,78 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSE4.1 and SSE4.2 instructions +@m_since_latest + +Equivalent to @cpp #include @ce and +@cpp #include @ce on most compilers except for GCC 4.8, where it +contains an additional workaround to make the instructions available with just +the @ref CORRADE_ENABLE_SSE41 and @ref CORRADE_ENABLE_SSE42 function attributes +instead of having to specify `-msse4.1` and `-msse4.2` for the whole +compilation unit. + +Because GCC puts both the SSE4.1 and the SSE4.2 instructions into the same +header and just guards each with a different macro, they have to be included +together, unlike with other SSE variants. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSse3.h, @ref IntrinsicsSsse3.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If I wouldn't need the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE2, SSE3 and SSSE3 headers explicitly + before, in correct order */ +#include "Corrade/Utility/IntrinsicsSsse3.h" + +/* Unfortunately, as opposed to AVX, the SSE4.1 and 4.2 intrinsics live in the + exact same header, which for GCC 4.8 means we have to subsequently have + target("sse4.2") again in order to call even SSE4.1. This is the reason why + CORRADE_ENABLE_SSE41 is not defined there. */ +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("sse4.1") +#pragma GCC target("sse4.2") +#pragma push_macro("__SSE4_1__") +#pragma push_macro("__SSE4_2__") +#ifndef __SSE4_1__ +#define __SSE4_1__ +#endif +#ifndef __SSE4_2__ +#define __SSE4_2__ +#endif +#endif +#include +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSE4_1__") +#pragma pop_macro("__SSE4_2__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSsse3.h b/src/Corrade/Utility/IntrinsicsSsse3.h new file mode 100644 index 000000000..c87fc7e9f --- /dev/null +++ b/src/Corrade/Utility/IntrinsicsSsse3.h @@ -0,0 +1,61 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +/** @file +@brief Intrinsics for x86 SSSE3 instructions +@m_since_latest + +Equivalent to @cpp #include @ce on most compilers except for GCC +4.8, where it contains an additional workaround to make the instructions +available with just the @ref CORRADE_ENABLE_SSSE3 function attribute instead of +having to specify `-mssse3` for the whole compilation unit. +@see @relativeref{Corrade,Cpu}, @ref Cpu-usage-target-attributes, + @ref IntrinsicsSse2.h, @ref IntrinsicsSse3.h, @ref IntrinsicsSse4.h, + @ref IntrinsicsAvx.h +*/ + +/* See https://gist.github.com/rygorous/f26f5f60284d9d9246f6 for more info. + If it wouldn't be for the #define, it could be all put into a macro with + _Pragma to wrap around the include, but because I do, there has to be one + wrapper header for each include. */ + +/* So users don't need to include the SSE2 and SSE3 headers explicitly before, + in correct order */ +#include "Corrade/Utility/IntrinsicsSse3.h" + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("ssse3") +#pragma push_macro("__SSSE3__") +#ifndef __SSSE3__ +#define __SSSE3__ +#endif +#endif +#include +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma pop_macro("__SSSE3__") +#pragma GCC pop_options +#endif From e602f577c5483d3399d539c57b0ef80d778c0ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 10 Jul 2022 14:35:26 +0200 Subject: [PATCH 09/79] Cpu: sigh clang-cl what are you doing. So you're telling me header size is a problem *with intrinsics*? Which are completely innocent easy-to-parse builtins, as opposed to template insanities in , and the like, only there's really many of them? Heh. Furthermore, I don't believe the Actually using intrinsics on Windows already requires the right /arch: settings, so this patch should have no big behavior change. part in the referenced commit, esepcially in contrast with common practice like in https://gist.github.com/rygorous/f26f5f60284d9d9246f6. For them in particular, GCC/Clang before functioning target attributes were the *worse* toolchain, because while in MSVC you could just use *any* intrinsics, on GCC you had to explicitly pass an -m option to use them. So I really don't believe this, plus the tests I have for the CORRADE_ENABLE_* macros all work, including AVX2, even though on the CIs MSVC compiles the file with baseline SSE2. If I get pissed I may do a similar #pragma push_macro() thingy like I did for GCC 4.8 with the IntrinsicsAvx.h header, but right now I'm not aware of any regular clang-cl users that would benefit from that. (If you are a clang-cl user and want this feature, let me know!) --- src/Corrade/Cpu.h | 71 +++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index c82c84ca8..a9a0eb97c 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1040,8 +1040,8 @@ constexpr Features runtimeFeatures() { return compiledFeatures(); } @brief Enable SSE2 for given function @m_since_latest -On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to -@cpp __attribute__((__target__("sse2"))) @ce, allowing use of +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("sse2"))) @ce, allowing use of [SSE2](https://en.wikipedia.org/wiki/SSE2) and earlier SSE instructions inside a function annotated with this macro without having to specify `-msse2` for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't @@ -1067,7 +1067,7 @@ more information and usage example. */ #if defined(CORRADE_TARGET_SSE2) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE2 -#elif defined(CORRADE_TARGET_GCC) +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE_SSE2 __attribute__((__target__("sse2"))) #elif defined(CORRADE_TARGET_MSVC) #define CORRADE_ENABLE_SSE2 @@ -1104,7 +1104,7 @@ Superset of @ref CORRADE_ENABLE_SSE2, implied by @ref CORRADE_ENABLE_SSSE3. See */ #if defined(CORRADE_TARGET_SSE3) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE3 -#elif defined(CORRADE_TARGET_GCC) +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -msse3 option implies -msse2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse3`) */ #define CORRADE_ENABLE_SSE3 __attribute__((__target__("sse3"))) @@ -1116,8 +1116,8 @@ Superset of @ref CORRADE_ENABLE_SSE2, implied by @ref CORRADE_ENABLE_SSSE3. See @brief Enable SSSE3 for given function @m_since_latest -On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to -@cpp __attribute__((__target__("ssse3"))) @ce, allowing use of +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("ssse3"))) @ce, allowing use of [SSSE3](https://en.wikipedia.org/wiki/SSSE3) and earlier SSE instructions inside a function annotated with this macro without having to specify `-mssse3` for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler @@ -1143,7 +1143,7 @@ Superset of @ref CORRADE_ENABLE_SSE3, implied by @ref CORRADE_ENABLE_SSE41. See */ #if defined(CORRADE_TARGET_SSSE3) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSSE3 -#elif defined(CORRADE_TARGET_GCC) +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -mssse3 option implies -msse2 -msse3 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mssse3`) */ #define CORRADE_ENABLE_SSSE3 __attribute__((__target__("ssse3"))) @@ -1155,8 +1155,8 @@ Superset of @ref CORRADE_ENABLE_SSE3, implied by @ref CORRADE_ENABLE_SSE41. See @brief Enable SSE4.1 for given function @m_since_latest -On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to -@cpp __attribute__((__target__("sse4.1"))) @ce, allowing use of +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("sse4.1"))) @ce, allowing use of [SSE4.1](https://en.wikipedia.org/wiki/SSE4#SSE4.1) and earlier SSE instructions inside a function annotated with this macro without having to specify `-msse4.1` for the whole compilation unit. On x86 MSVC expands to @@ -1182,7 +1182,7 @@ See @ref Cpu-usage-target-attributes for more information and usage example. */ #if defined(CORRADE_TARGET_SSE41) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE41 -#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) +#elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 409) || defined(CORRADE_TARGET_CLANG) /* also matches clang-cl */ /* The -msse4.1 option implies -msse2 -msse3 -mssse3 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse4.1`) */ @@ -1195,8 +1195,8 @@ See @ref Cpu-usage-target-attributes for more information and usage example. @brief Enable SSE4.2 for given function @m_since_latest -On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to -@cpp __attribute__((__target__("sse4.2"))) @ce, allowing use of +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("sse4.2"))) @ce, allowing use of [SSE4.2](https://en.wikipedia.org/wiki/SSE4#SSE4.2) and earlier SSE instructions inside a function annotated with this macro without having to specify `-msse4.2` for the whole compilation unit. On x86 MSVC expands to @@ -1222,7 +1222,7 @@ Superset of @ref CORRADE_ENABLE_SSE41, implied by @ref CORRADE_ENABLE_AVX. See */ #if defined(CORRADE_TARGET_SSE42) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE42 -#elif defined(CORRADE_TARGET_GCC) +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -msse4.2 option implies -msse2 -msse3 -mssse3 -msse4.1 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse4.2`) */ @@ -1240,8 +1240,10 @@ On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to [AVX](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) and all earlier SSE instructions inside a function annotated with this macro without having to specify `-mavx` for the whole compilation unit. On x86 MSVC expands to nothing, -as the compiler doesn't restrict use of intrinsics in any way. Not defined on -other compilers or architectures. +as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE +variants this macro is not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", +as there AVX and newer intrinsics are provided only if enabled on compiler +command line. Not defined on other compilers or architectures. As a special case, if @ref CORRADE_TARGET_AVX is present (meaning AVX is enabled for the whole compilation unit), this macro is defined as empty on all @@ -1262,12 +1264,17 @@ Superset of @ref CORRADE_ENABLE_SSE42, implied by @ref CORRADE_ENABLE_AVX2. See */ #if defined(CORRADE_TARGET_AVX) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX -#elif defined(CORRADE_TARGET_GCC) +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ /* The -mavx option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx`) */ #define CORRADE_ENABLE_AVX __attribute__((__target__("avx"))) -#elif defined(CORRADE_TARGET_MSVC) +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __AVX__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE_AVX #endif @@ -1281,7 +1288,10 @@ On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to FMA, F16C, AVX and all earlier SSE instructions inside a function annotated with this macro without having to specify `-mavx2` for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of -intrinsics in any way. Not defined on other compilers or architectures. +intrinsics in any way. Unlike the SSE variants this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer intrinsics are +provided only if enabled on compiler command line. Not defined on other +compilers or architectures. As a special case, if @ref CORRADE_TARGET_AVX2 is present (meaning AVX2 is enabled for the whole compilation unit), this macro is defined as empty on all @@ -1302,12 +1312,17 @@ See @ref Cpu-usage-target-attributes for more information and usage example. */ #if defined(CORRADE_TARGET_AVX2) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX2 -#elif defined(CORRADE_TARGET_GCC) +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ /* The -mavx2 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx2`) */ #define CORRADE_ENABLE_AVX2 __attribute__((__target__("avx2"))) -#elif defined(CORRADE_TARGET_MSVC) +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __AVX2__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE_AVX2 #endif @@ -1321,8 +1336,11 @@ On @ref CORRADE_TARGET_X86 "x86" GCC 4.9+ and Clang expands to and SSE instructions inside a function annotated with this macro without having to specify `-mavx512f` for the whole compilation unit. On x86 MSVC 2017 15.3+ expands to nothing, as the compiler doesn't restrict use of intrinsics in any -way. Not defined on other compilers, earlier compiler versions without AVX-512 -support or other architectures. +way. Unlike the SSE variants this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer intrinsics are +provided only if enabled on compiler command line. Not defined on other +compilers, earlier compiler versions without AVX-512 support or other +architectures. As a special case, if @ref CORRADE_TARGET_AVX512F is present (meaning AVX-512 Foundation is enabled for the whole compilation unit), this macro is defined as @@ -1334,12 +1352,17 @@ more information and usage example. */ #if defined(CORRADE_TARGET_AVX512F) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX512F -#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ /* The -mavx512 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx -mavx2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx512f`) */ #define CORRADE_ENABLE_AVX512F __attribute__((__target__("avx512f"))) -#elif defined(CORRADE_TARGET_MSVC) && _MSC_VER >= 1911 +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __AVX512F__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && _MSC_VER >= 1911 && !defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE_AVX512F #endif #endif From 23c4fb2c23ef371e354b9c83cf856ceab14bd864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 10 Jul 2022 01:09:46 +0200 Subject: [PATCH 10/79] Cpu: ARM and WebAssembly CORRADE_ENABLE_* function attribute macros. The situation is ... less great with these. What bothers me the most is Clang on ARM, where it looks like function target attributes and runtime dispatch would be impossible to implement. Pretty bad, given it's the compiler of choice on Android, iOS and now macOS as well. --- src/Corrade/Corrade.h | 12 ++-- src/Corrade/Cpu.h | 135 +++++++++++++++++++++++++++++++++-- src/Corrade/Test/CpuTest.cpp | 89 +++++++++++++++++++++++ 3 files changed, 228 insertions(+), 8 deletions(-) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index e9c148735..fb504bd56 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -712,7 +712,8 @@ sets. Detection for these will be added when the platforms become more successor to both NEON and SVE, scheduled to appear in production in around 2022 -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Neon} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Neon}, + @ref CORRADE_ENABLE_NEON */ #define CORRADE_TARGET_NEON #undef CORRADE_TARGET_NEON @@ -726,7 +727,8 @@ at compile time (`-mfpu=neon-vfpv4` on GCC/Clang on 32-bit ARM, implicitly supported on ARM64). Not defined if FMA is only available for scalar code and not for NEON. Superset of @ref CORRADE_TARGET_NEON, implied by @ref CORRADE_TARGET_NEON_FP16. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFma} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFma}, + @ref CORRADE_ENABLE_NEON_FMA */ #define CORRADE_TARGET_NEON_FMA #undef CORRADE_TARGET_NEON_FMA @@ -738,7 +740,8 @@ not for NEON. Superset of @ref CORRADE_TARGET_NEON, implied by Defined on @ref CORRADE_TARGET_ARM "ARM" if ARMv8.2-a NEON FP16 vector arithmetic support is enabled at compile time (`-march=armv8.2-a+fp16` on GCC/Clang). Superset of @ref CORRADE_TARGET_NEON_FMA. -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFp16} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::NeonFp16}, + @ref CORRADE_ENABLE_NEON_FP16 */ #define CORRADE_TARGET_NEON_FP16 #undef CORRADE_TARGET_NEON_FP16 @@ -750,7 +753,8 @@ GCC/Clang). Superset of @ref CORRADE_TARGET_NEON_FMA. Defined on @ref CORRADE_TARGET_WASM "WebAssembly" if [128-bit SIMD](https://github.com/webassembly/simd) instructions are enabled at compile time (`-msimd128` passed to Clang). -@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Simd128} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Simd128}, + @ref CORRADE_ENABLE_SIMD128 */ #define CORRADE_TARGET_SIMD128 #undef CORRADE_TARGET_SIMD128 diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index a9a0eb97c..333c8dac2 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -591,7 +591,7 @@ constexpr Avx512fT Avx512f{Implementation::Init}; [ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)). Available only on @ref CORRADE_TARGET_ARM "ARM". Superset of @ref Scalar, implied by @ref NeonFp16. -@see @ref CORRADE_TARGET_NEON +@see @ref CORRADE_TARGET_NEON, @ref CORRADE_ENABLE_NEON */ constexpr NeonT Neon{Implementation::Init}; @@ -601,7 +601,7 @@ constexpr NeonT Neon{Implementation::Init}; [ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) with FMA instructions. Available only on @ref CORRADE_TARGET_ARM "ARM". Superset of @ref Neon, implied by @ref NeonFp16. -@see @ref CORRADE_TARGET_NEON_FMA +@see @ref CORRADE_TARGET_NEON_FMA, @ref CORRADE_ENABLE_NEON_FMA */ constexpr NeonFmaT NeonFma{Implementation::Init}; @@ -611,7 +611,7 @@ constexpr NeonFmaT NeonFma{Implementation::Init}; [ARM NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) with ARMv8.2-a FP16 vector arithmetic. Available only on @ref CORRADE_TARGET_ARM "ARM". Superset of @ref NeonFma. -@see @ref CORRADE_TARGET_NEON_FP16 +@see @ref CORRADE_TARGET_NEON_FP16, @ref CORRADE_ENABLE_NEON_FP16 */ constexpr NeonFp16T NeonFp16{Implementation::Init}; #endif @@ -622,7 +622,7 @@ constexpr NeonFp16T NeonFp16{Implementation::Init}; [128-bit WebAssembly SIMD](https://github.com/webassembly/simd). Available only on @ref CORRADE_TARGET_WASM "WebAssembly". Superset of @ref Scalar. -@see @ref CORRADE_TARGET_SIMD128 +@see @ref CORRADE_TARGET_SIMD128, @ref CORRADE_ENABLE_SIMD128 */ constexpr Simd128T Simd128{Implementation::Init}; #endif @@ -1367,6 +1367,133 @@ more information and usage example. #endif #endif +#if defined(CORRADE_TARGET_ARM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief Enable NEON for given function +@m_since_latest + +On 32-bit @ref CORRADE_TARGET_ARM "ARM" GCC expands to +@cpp __attribute__((__target__("fpu=neon"))) @ce, allowing use of +[NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +instructions inside a function annotated with this macro without having to +specify `-mfpu=neon` for the whole compilation unit. On ARM MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. In +contrast to GCC, this macro is not defined on Clang, as it makes the NEON +intrinsics available only if enabled on compiler command line. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_NEON is present (meaning NEON is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. This is also the case for ARM64, where NEON support is implicit +(and where `-mfpu=neon` is unrecognized). + +Implied by @ref CORRADE_ENABLE_NEON_FMA. See @ref Cpu-usage-target-attributes +for more information and usage example. +@see @relativeref{Corrade,Cpu::Neon} +*/ +#if defined(CORRADE_TARGET_NEON) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON +/* https://github.com/android/ndk/issues/1066 is the only reported (and + ignored) issue I found, feels strange that people would just not use ifunc + or target attributes on Android at all and instead put everything in + separate files. Needs further investigation. Too bad most ARM platforms + ditched GCC, where this works properly. */ +#elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#define CORRADE_ENABLE_NEON __attribute__((__target__("fpu=neon"))) +#elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON +#endif + +/** +@brief Enable NEON FMA for given function +@m_since_latest + +On 32-bit @ref CORRADE_TARGET_ARM "ARM" GCC expands to +@cpp __attribute__((__target__("fpu=neon-vfpv4"))) @ce, allowing use of +[NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) FMA +instructions inside a function annotated with this macro without having to +specify `-mfpu=neon-vfpv4` for the whole compilation unit. On ARM MSVC expands +to nothing, as the compiler doesn't restrict use of intrinsics in any way. In +contrast to GCC, this macro is not defined on Clang, as it makes the NEON FMA +intrinsics available only if enabled on compiler command line. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_NEON_FMA is present (meaning NEON FMA +is enabled for the whole compilation unit), this macro is defined as empty on +all compilers. This is also the case for ARM64, where NEON support is implicit +(and where `-mfpu=neon-vfpv4` is unrecognized). + +Superset of @ref CORRADE_ENABLE_NEON, implied by @ref CORRADE_ENABLE_NEON_FP16. +See @ref Cpu-usage-target-attributes for more information and usage example. +@see @relativeref{Corrade,Cpu::NeonFma} +*/ +#if defined(CORRADE_TARGET_NEON_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FMA +/* See CORRADE_ENABLE_NEON above for details about Clang */ +#elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#define CORRADE_ENABLE_NEON_FMA __attribute__((__target__("fpu=neon-vfpv4"))) +#elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FMA +#endif + +/** +@brief Enable NEON FP16 for given function +@m_since_latest + +On @ref CORRADE_TARGET_ARM "ARM" GCC expands to +@cpp __attribute__((__target__("arch=armv8.2-a+fp16"))) @ce, allowing use of +ARMv8.2-a [NEON](https://en.wikipedia.org/wiki/ARM_architecture#Advanced_SIMD_(Neon)) +FP16 vector arithmetic inside a function annotated with this macro without +having to specify `-march=armv8.2-a+fp16` for the whole compilation unit. On +ARM MSVC expands to nothing, as the compiler doesn't restrict use of intrinsics +in any way. In contrast to GCC, this macro is not defined on Clang, as it makes +the NEON FP16 intrinsics available only if enabled on compiler command line. +Not defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_NEON_FP16 is present (meaning NEON +FP16 is enabled for the whole compilation unit), this macro is defined as empty +on all compilers. + +Superset of @ref CORRADE_ENABLE_NEON_FMA. See @ref Cpu-usage-target-attributes +for more information and usage example. +@see @relativeref{Corrade,Cpu::NeonFp16} +*/ +#if defined(CORRADE_TARGET_NEON_FP16) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FP16 +/* See CORRADE_ENABLE_NEON above for details about Clang */ +#elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#define CORRADE_ENABLE_NEON_FP16 __attribute__((__target__("arch=armv8.2-a+fp16"))) +#elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_NEON_FP16 +#endif +#endif + +#if defined(CORRADE_TARGET_WASM) || defined(DOXYGEN_GENERATING_OUTPUT) +/** +@brief Enable SIMD128 for given function +@m_since_latest + +Given that it's currently not possible to selectively use +[128-bit SIMD](https://github.com/webassembly/simd) in a WebAssembly module +without causing a compilation error on runtimes that don't support it, this +macro is only defined if @ref CORRADE_TARGET_SIMD128 is present (meaning +SIMD128 is explicitly enabled for the whole compilation unit), and is always +empty, as @cpp __attribute__((__target__("simd128"))) @ce would be redundant +if `-msimd128` is passed on the command line. + +The situation may change once the +[feature detection proposal](https://github.com/WebAssembly/feature-detection/blob/main/proposals/feature-detection/Overview.md) +is implemented, but likely only for instruction sets building on top of this +one. + +See @ref Cpu-usage-target-attributes for more information and usage example. +@see @relativeref{Corrade,Cpu::Simd128} +*/ +#if defined(CORRADE_TARGET_SIMD128) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_SIMD128 +#endif +#endif + } } diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index b53645633..c332c2fe6 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -46,6 +46,12 @@ #if defined(CORRADE_ENABLE_AVX) || defined(CORRADE_ENABLE_AVX_F16C) || defined(CORRADE_ENABLE_AVX_FMA) || defined(CORRADE_ENABLE_AVX2) || defined(CORRADE_ENABLE_AVX512F) #include "Corrade/Utility/IntrinsicsAvx.h" #endif +#ifdef CORRADE_ENABLE_NEON +#include +#endif +#ifdef CORRADE_ENABLE_SIMD128 +#include +#endif namespace Corrade { namespace Test { namespace { @@ -126,6 +132,12 @@ CpuTest::CpuTest() { &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, + #elif defined(CORRADE_TARGET_ARM) + &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, + #elif defined(CORRADE_TARGET_WASM) + &CpuTest::enableMacros, #endif &CpuTest::debug, @@ -642,6 +654,83 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX512F int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON int callInstructionFor() { + int32x4_t a{-10, 20, -30, 40}; + + /* All instructions NEON */ + union { + int32x4_t v; + int s[8]; + } b; + b.v = vabsq_s32(a); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_NEON_FMA +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON_FMA int callInstructionFor() { + /* Values from Magnum::Math::Test::FunctionsTest::fma() */ + float32x4_t a{0.0f, 2.0f, 1.5f, 0.5f}; + float32x4_t b{0.0f, 3.0f, 2.0f, -1.0f}; + float32x4_t c{0.0f, 0.75f, 0.25f, 0.1f}; + + /* FMA */ + union { + float32x4_t v; + float s[4]; + } d; + d.v = vfmaq_f32(c, b, a); + + CORRADE_COMPARE(d.s[0], 0.0f); + CORRADE_COMPARE(d.s[1], 6.75f); + CORRADE_COMPARE(d.s[2], 3.25f); + CORRADE_COMPARE(d.s[3], -0.4f); + return d.s[2]; +} +#endif +#ifdef CORRADE_ENABLE_NEON_FP16 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON_FP16 int callInstructionFor() { + float32x4_t a{5.47f, 2.23f, 7.62f, 0.5f}; + float16x4_t b = vcvt_f16_f32(a); + + /* FP16 */ + float16x4_t c = vrndp_f16(b); + + union { + float32x4_t v; + float s[4]; + } d; + d.v = vcvt_f32_f16(c); + CORRADE_COMPARE(d.s[0], 6.0f); + CORRADE_COMPARE(d.s[1], 3.0f); + CORRADE_COMPARE(d.s[2], 8.0f); + CORRADE_COMPARE(d.s[3], 1.0f); + return d.s[3]; +} +#endif +#ifdef CORRADE_ENABLE_SIMD128 +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SIMD128 int callInstructionFor() { + v128_t a = wasm_i32x4_make(-10, 20, -30, 40); + + /* All instructions SIMD128 */ + union { + v128_t v; + int s[8]; + } b; + b.v = wasm_i32x4_abs(a); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; +} +#endif template void CpuTest::enableMacros() { setTestCaseTemplateName(Cpu::TypeTraits::name()); From c8a43c9f1da55d535a478a8aa2a44a3bf0c9b678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 10 Jul 2022 02:05:52 +0200 Subject: [PATCH 11/79] Cpu: implement runtime ARM feature detection on Linux and Android 18+. What is interesting that I successfully managed to use the FP16 instructions on my 2017 phone that ... doesn't say it supports ASIMDHP. Strange. --- src/Corrade/Cpu.cpp | 65 +++++++++++++++++++++++++++++++++++++++++---- src/Corrade/Cpu.h | 21 ++++++++++++--- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index 2b8cf58ab..1595d1d98 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -53,6 +53,11 @@ #include #endif #endif + +/* ARM on Linux and Android with API level 18+ */ +#elif defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) +#include +#include #endif namespace Corrade { namespace Cpu { @@ -79,7 +84,50 @@ static_assert(sizeof(Cpu::NeonFp16) == 1, ""); static_assert(sizeof(Cpu::Simd128) == 1, ""); #endif -#if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) +#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) +namespace Implementation { + +/* As getauxval() can't be called from within an ifunc resolver because there + it's too early for an external call, the value of AT_HWCAP is instead passed + to it from outside, on glibc 2.13+ and on Android API 30+: + https://github.com/bminor/glibc/commit/7520ff8c744a704ca39741c165a2360d63a4f47a + https://android.googlesource.com/platform/bionic/+/e949195f6489653ee3771535951ed06973246c3e/libc/include/sys/ifunc.h + Which means we need a variant of runtimeFeatures() that is able to operate + with a value fed from outside, which is then used inside such resolvers. + For simplicity this variant is available always and the public + Cpu::runtimeFeatures() just delegates to it. */ +Features runtimeFeatures(const unsigned long caps) { + unsigned int out = 0; + #ifdef CORRADE_TARGET_32BIT + if(caps & HWCAP_NEON) out |= TypeTraits::Index; + /* Since FMA is enabled by passing -mfpu=neon-vfpv4, I assume this is the + flag that corresponds to it. */ + if(caps & HWCAP_VFPv4) out |= TypeTraits::Index; + #else + /* On ARM64 NEON and NEON FMA is implicit. For extra security make use of + the CORRADE_TARGET_ defines (which should be always there). */ + out |= + #ifdef CORRADE_TARGET_NEON + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FMA + TypeTraits::Index| + #endif + 0; + /* The HWCAP flags are extremely cryptic. The only vague confirmation is in + a *commit message* to the kernel hwcaps file, FFS. The HWCAP_FPHP seems + to correspond to scalar FP16, so the other should be the vector one? + https://github.com/torvalds/linux/blame/master/arch/arm64/include/uapi/asm/hwcap.h + This one also isn't present on 32-bit, so I assume it's ARM64-only? */ + if(caps & HWCAP_ASIMDHP) out |= TypeTraits::Index; + #endif + return Features{out}; +} + +} +#endif + +#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) #if defined(CORRADE_TARGET_X86) && defined(CORRADE_TARGET_GCC) /* GCC 10 complains otherwise, however 4.8 doesn't. Doing it via a function attribute instead of passing -mxsave as detecting target architecture in @@ -87,8 +135,6 @@ static_assert(sizeof(Cpu::Simd128) == 1, ""); __attribute__((__target__("xsave"))) #endif Features runtimeFeatures() { - unsigned int out = 0; - /* Use cpuid on x86 and GCC/Clang/MSVC */ #if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) union { @@ -109,6 +155,7 @@ Features runtimeFeatures() { CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid(1, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); #endif + unsigned int out = 0; if(cpuid.e.dx & (1 << 26)) out |= TypeTraits::Index; if(cpuid.e.cx & (1 << 0)) out |= TypeTraits::Index; if(cpuid.e.cx & (1 << 9)) out |= TypeTraits::Index; @@ -170,13 +217,21 @@ Features runtimeFeatures() { if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; + return Features{out}; + + /* Use getauxval() on ARM on Linux and Android */ + #elif defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) + /* People say getauxval() is "extremely slow": + https://lemire.me/blog/2020/07/17/the-cost-of-runtime-dispatch/#comment-538459 + Like, can anything be worse than reading and parsing the text from + /proc/cpuinfo? */ + return Implementation::runtimeFeatures(getauxval(AT_HWCAP)); + /* No other implementation at the moment. The function should not be even defined here in that case -- it's inlined in the header instead. */ #else #error #endif - - return Features{out}; } #endif diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 333c8dac2..2e8e1a57b 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -720,6 +720,14 @@ Returns a tag corresponding to tag type @p T. The following two expressions are */ template constexpr T tag() { return T{Implementation::Init}; } +#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) +class Features; + +namespace Implementation { + CORRADE_UTILITY_EXPORT Features runtimeFeatures(unsigned long caps); +} +#endif + /** @brief Feature set @@ -835,6 +843,9 @@ class Features { /* MSVC demands the export macro to be here as well */ friend CORRADE_UTILITY_EXPORT Features runtimeFeatures(); #endif + #if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) + friend Features Implementation::runtimeFeatures(unsigned long); + #endif constexpr explicit Features(unsigned int data) noexcept: _data{data} {} @@ -1012,9 +1023,11 @@ not present, no following flags are checked either. On compilers other than GCC, Clang and MSVC the function is @cpp constexpr @ce and delegates into @ref compiledFeatures(). -On @ref CORRADE_TARGET_ARM "ARM", no runtime detection is implemented at the -moment. The function is @cpp constexpr @ce and delegates into -@ref compiledFeatures(). +On @ref CORRADE_TARGET_ARM "ARM" and Linux or Android API level 18+ uses +@m_class{m-doc-external} [getauxval()](https://man.archlinux.org/man/getauxval.3) +to check for the @ref Neon, @ref NeonFma and @ref NeonFp16. @ref Neon and +@ref NeonFma are implicitly supported on ARM64. On other platforms the function +is @cpp constexpr @ce and delegates into @ref compiledFeatures(). On @ref CORRADE_TARGET_WASM "WebAssembly" an attempt to use SIMD instructions without runtime support results in a WebAssembly compilation error and thus @@ -1029,7 +1042,7 @@ default-constructed) @ref Features. @see @ref DefaultBase */ -#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || defined(DOXYGEN_GENERATING_OUTPUT) +#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(DOXYGEN_GENERATING_OUTPUT) CORRADE_UTILITY_EXPORT Features runtimeFeatures(); #else constexpr Features runtimeFeatures() { return compiledFeatures(); } From 0d6a8d86c93780f3ec55fcb4dd5859205b2c8e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 13 Jul 2022 22:06:14 +0200 Subject: [PATCH 12/79] doc: setup Doxygen for dot graphs. Ye, Corrade got so complicated that I need them here as well. --- doc/Doxyfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/Doxyfile b/doc/Doxyfile index 50452ea75..cc77f226b 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -2280,14 +2280,14 @@ DOT_NUM_THREADS = 0 # The default value is: Helvetica. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Verdana +DOT_FONTNAME = Source Sans Pro # The DOT_FONTSIZE tag can be used to set the size (in points) of the font of # dot graphs. # Minimum value: 4, maximum value: 24, default value: 10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 8 +DOT_FONTSIZE = 16 # By default doxygen will tell dot to use the default font as specified with # DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set @@ -2445,7 +2445,7 @@ DOT_PATH = # command). # This tag requires that the tag HAVE_DOT is set to YES. -DOTFILE_DIRS = +DOTFILE_DIRS = . # The MSCFILE_DIRS tag can be used to specify one or more directories that # contain msc files that are included in the documentation (see the \mscfile From 099461f392504e1a25fdad39face5355a8758c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 13 Jul 2022 22:57:00 +0200 Subject: [PATCH 13/79] Cpu: implement dispatching for extra tags not in the base hierarchy. This feature took me SIXTEEN MONTHS to implement, literally. No amount of multiple inheritance, virtual diamond inheritance, manually written combinatorically exploding classes with custom conversion operations did help. The only, although slightly ugly, solution is to dispatch with *two* function parameters, where one controls the sanity (that the requested feature set is same or a superset of what the overload needs), and the other controls the priority. With a calculated chain of base classes, because yes that's still the core idea. Besides that: - The tags can be now ORed, ANDed etc. like before, but now they produce the value in a type, so the type can be used in compile-time pverload resolution. That type is convertible to the usual Cpu::Features so I didn't think it was necessary to even let the users know there's some other type going on. - To actually implement the dispatch with *something*, four new tags got added -- AVX F16C, AVX FMA, POPCNT and LZCNT. I discovered a lot of new old x86 warts and GCC 4.8 issues when implementing these. Fun times. - To hide the two-parameter overload resolution complexity, there's two new macros, CORRADE_CPU_DECLARE() and CORRADE_CPU_SELECT(), one for function declarations, one for actually calling such declarations. Names hopefully self-explanatory, first I tried with CORRADE_CPU_TYPE() and CORRADE_CPU() and that was shorter, but quite confusing. - The docs are hopefully concrete enough to make it clear what's going on with this feature, but not detailed enough to reveal the mess underneath. --- doc/cpu.dot | 56 +++ doc/snippets/Corrade-cpp17.cpp | 10 + doc/snippets/Corrade.cpp | 48 +- src/Corrade/Corrade.h | 25 +- src/Corrade/Cpu.cpp | 82 ++- src/Corrade/Cpu.h | 713 ++++++++++++++++++++++++++- src/Corrade/Test/CpuTest.cpp | 360 +++++++++++++- src/Corrade/Utility/IntrinsicsAvx.h | 53 +- src/Corrade/Utility/IntrinsicsSse4.h | 19 +- 9 files changed, 1296 insertions(+), 70 deletions(-) create mode 100644 doc/cpu.dot diff --git a/doc/cpu.dot b/doc/cpu.dot new file mode 100644 index 000000000..215863149 --- /dev/null +++ b/doc/cpu.dot @@ -0,0 +1,56 @@ +/* + This file is part of Magnum. + + Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, + 2020, 2021, 2022 Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +digraph "x86 instruction family tree" { + rankdir=RL + ranksep=0.5 + + node [shape=circle style=filled margin="0.01,0.01" width=1] + + SSE2 [class="m-success"] + SSE3 [class="m-success"] + SSSE3 [class="m-success"] + SSE41 [class="m-success" label="SSE4.1"] + SSE42 [class="m-success" label="SSE4.2"] + AVX [class="m-success"] + AVX2 [class="m-success"] + AVX512F [class="m-success"] + + FMA3 [width=0.75 fontsize=13 class="m-primary"] + F16C [width=0.75 fontsize=13 class="m-primary"] + POPCNT [width=0.75 fontsize=13 margin="0,0" class="m-primary"] + LZCNT [width=0.75 fontsize=13 class="m-primary"] + + POPCNT -> SSE3 [style=invis] + LZCNT -> SSE3 [style=invis] + LZCNT -> POPCNT [style=invis] + + FMA3 -> AVX [class="m-primary"] + F16C -> AVX [class="m-primary"] + + FMA3 -> F16C [style=invis] + + AVX512F -> AVX2 -> AVX -> SSE42 -> SSE41 -> SSSE3 -> SSE3 -> SSE2 [class="m-success" weight=100] +} diff --git a/doc/snippets/Corrade-cpp17.cpp b/doc/snippets/Corrade-cpp17.cpp index 688821c89..02f23a073 100644 --- a/doc/snippets/Corrade-cpp17.cpp +++ b/doc/snippets/Corrade-cpp17.cpp @@ -42,5 +42,15 @@ if constexpr(Cpu::DefaultBase >= Cpu::Avx2) { // scalar code } /* [Cpu-usage-compile-time] */ + +/* [Cpu-usage-extra-compile-time] */ +Utility::Debug{} << "Base and extra instruction sets:" << Cpu::Default; + +if constexpr(Cpu::Default >= (Cpu::Avx2|Cpu::AvxFma)) { + // AVX2+FMA code +} else { + // scalar code +} +/* [Cpu-usage-extra-compile-time] */ } #endif diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index 98cbf0ff7..48fbd8409 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -39,18 +39,43 @@ void transform(Cpu::Sse42T, Containers::ArrayView data); void transform(Cpu::Avx2T, Containers::ArrayView data); /* [Cpu-usage-declare] */ +/* [Cpu-usage-extra-declare] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse2), DOXYGEN_ELLIPSIS(int)); +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)); +/* [Cpu-usage-extra-declare] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Scalar), DOXYGEN_ELLIPSIS(int)); +/* Might needed if Default doesn't include SSE2 on 32-bit */ + +namespace Foo { +/* [Cpu-usage-extra-ambiguity] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt), DOXYGEN_ELLIPSIS(int)); +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)); +/* [Cpu-usage-extra-ambiguity] */ + +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), int); +/* [Cpu-usage-extra-ambiguity-resolve] */ +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)) { + // Or the other variant, or a custom third implementation ... + return lookup(CORRADE_CPU_SELECT(Cpu::Sse41|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(0)); +} +/* [Cpu-usage-extra-ambiguity-resolve] */ +} + /* [Cpu-usage-target-attributes] */ -void transform(Cpu::ScalarT, Containers::ArrayView data) { - DOXYGEN_ELLIPSIS(static_cast(data);) +int lookup(CORRADE_CPU_DECLARE(Cpu::Scalar), DOXYGEN_ELLIPSIS(int)) { + DOXYGEN_ELLIPSIS(return 0;) } -#ifdef CORRADE_ENABLE_SSE42 -CORRADE_ENABLE_SSE42 void transform(Cpu::Sse42T, Containers::ArrayView data) { - DOXYGEN_ELLIPSIS(static_cast(data);) +#ifdef CORRADE_ENABLE_SSE2 +CORRADE_ENABLE_SSE2 int lookup(CORRADE_CPU_DECLARE(Cpu::Sse2), DOXYGEN_ELLIPSIS(int)) { + DOXYGEN_ELLIPSIS(return 0;) } #endif -#ifdef CORRADE_ENABLE_AVX2 -CORRADE_ENABLE_AVX2 void transform(Cpu::Avx2T, Containers::ArrayView data) { - DOXYGEN_ELLIPSIS(static_cast(data);) +#if defined(CORRADE_ENABLE_SSE41) && \ + defined(CORRADE_ENABLE_POPCNT) && \ + defined(CORRADE_ENABLE_LZCNT) +CORRADE_ENABLE_SSE41 CORRADE_ENABLE_POPCNT CORRADE_ENABLE_LZCNT +int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)) { + DOXYGEN_ELLIPSIS(return 0;) } #endif /* [Cpu-usage-target-attributes] */ @@ -82,6 +107,13 @@ else /* [Cpu-usage-runtime-manual-dispatch] */ } +{ +/* [Cpu-usage-extra-compile-time-call] */ +int found = lookup(CORRADE_CPU_SELECT(Cpu::Default), DOXYGEN_ELLIPSIS(0)); +/* [Cpu-usage-extra-compile-time-call] */ +static_cast(found); +} + { /* [Cpu-tag-from-type] */ foo(Cpu::Avx2); diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index fb504bd56..a9ed19bb4 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -598,8 +598,10 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if [POPCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) is enabled at compile time. On GCC/Clang it's `-mpopcnt` and is also implied by `-msse4.2` and higher, MSVC doesn't have a direct option but it's assumed to -be implied by `/arch:AVX`. -@see @relativeref{Corrade,Cpu} +be implied by `/arch:AVX`. To avoid failures at runtime, prefer to detect its +presence with @ref Cpu::runtimeFeatures(). +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Popcnt}, + @ref CORRADE_ENABLE_POPCNT */ #define CORRADE_TARGET_POPCNT #undef CORRADE_TARGET_POPCNT @@ -613,8 +615,11 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if is enabled at compile time (on GCC/Clang it's `-mlznct`, MSVC doesn't have a direct option but it's assumed to be implied by `/arch:AVX`). However note that this instruction has encoding compatible with an earlier `BSR` instruction -which has a slightly different behavior. -@see @relativeref{Corrade,Cpu} +which has a slightly different behavior. To avoid wrong results if it isn't +available, prefer to detect its presence with @ref Cpu::runtimeFeatures() +instead. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Lzcnt}, + @ref CORRADE_ENABLE_LZCNT */ #define CORRADE_TARGET_LZCNT #undef CORRADE_TARGET_LZCNT @@ -641,8 +646,10 @@ MSVC). Superset of @ref CORRADE_TARGET_SSE42, implied by Defined on @ref CORRADE_TARGET_X86 "x86" if the [F16C instruction set](https://en.wikipedia.org/wiki/F16C) is enabled at compile time. On GCC/Clang it's `-mf16c`, MSVC doesn't have a direct option but -it's assumed to be implied by `/arch:AVX2`. -@see @relativeref{Corrade,Cpu} +it's assumed to be implied by `/arch:AVX2`. To avoid failures at runtime, +prefer to detect its presence with @ref Cpu::runtimeFeatures(). +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::AvxF16c}, + @ref CORRADE_ENABLE_AVX_F16C */ #define CORRADE_TARGET_AVX_F16C #undef CORRADE_TARGET_AVX_F16C @@ -654,12 +661,14 @@ it's assumed to be implied by `/arch:AVX2`. Defined on @ref CORRADE_TARGET_X86 "x86" if the [FMA3 instruction set](https://en.wikipedia.org/wiki/FMA_instruction_set) is enabled at compile time. On GCC/Clang it's `-mfma`, MSVC doesn't have a direct -option but it's assumes to be implied by `/arch:AVX2`. +option but it's assumes to be implied by `/arch:AVX2`. To avoid failures at +runtime, prefer to detect its presence with @ref Cpu::runtimeFeatures(). The FMA4 instruction set, which used to be supported only in certain range of AMD processors and isn't anymore, is not detected, and AMD switched to FMA3 since. -@see @relativeref{Corrade,Cpu} +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::AvxFma}, + @ref CORRADE_ENABLE_AVX_FMA */ #define CORRADE_TARGET_AVX_FMA #undef CORRADE_TARGET_AVX_FMA diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index 1595d1d98..b58dde0d2 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -74,6 +74,8 @@ static_assert(sizeof(Cpu::Ssse3) == 1, ""); static_assert(sizeof(Cpu::Sse41) == 1, ""); static_assert(sizeof(Cpu::Sse42) == 1, ""); static_assert(sizeof(Cpu::Avx) == 1, ""); +static_assert(sizeof(Cpu::AvxF16c) == 1, ""); +static_assert(sizeof(Cpu::AvxFma) == 1, ""); static_assert(sizeof(Cpu::Avx2) == 1, ""); static_assert(sizeof(Cpu::Avx512f) == 1, ""); #elif defined(CORRADE_TARGET_ARM) @@ -162,6 +164,13 @@ Features runtimeFeatures() { if(cpuid.e.cx & (1 << 19)) out |= TypeTraits::Index; if(cpuid.e.cx & (1 << 20)) out |= TypeTraits::Index; + /* https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits, + bit 5 says "ABM (lzcnt and popcnt)", but + https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation) + says that while LZCNT is advertised in the ABM CPUID bit, POPCNT is a + separate CPUID flag. Get POPCNT first, ABM later. */ + if(cpuid.e.cx & (1 << 23)) out |= TypeTraits::Index; + /* AVX needs OS support checked, as the OS needs to be capable of saving and restoring the expanded registers when switching contexts: https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Operating_system_support */ @@ -186,36 +195,49 @@ Features runtimeFeatures() { } /* If AVX is not supported, we don't check any following flags either */ - if(!(out & TypeTraits::Index)) return Features{out}; + if(out & TypeTraits::Index) { + if(cpuid.e.cx & (1 << 29)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 12)) out |= TypeTraits::Index; + + /* https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features */ + #ifdef CORRADE_TARGET_MSVC + __cpuidex(cpuid.data, 7, 0); + /* __get_cpuid_count() is only since GCC 7.1 and Clang 5.0: + https://github.com/gcc-mirror/gcc/commit/2488ebe5ef1788616c2fbc61e05af09f0749ebbe + https://github.com/llvm/llvm-project/commit/f6e8408a116405d0cc25b506e8c5b60ab4ab7bcd + Furthermore, Clang 5.0 is 9.3 on Apple, 9.2 has only Clang 4: + https://en.wikipedia.org/wiki/Xcode#Toolchain_versions */ + #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 701) || (defined(CORRADE_TARGET_CLANG) && ((!defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__ >= 5) || (defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__*100 + __clang_minor__ >= 903))) + CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid_count(7, 0, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); + /* Looking at the above commits, on older GCC, __get_cpuid(7) seems to + do what __get_cpuid_count(7, 0) does on new GCC, however that's only + since 6.3: + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 + Older Clang doesn't do any of that, and thus to avoid a + combinatorial explosion of test scenarios, we'll just directly use + the __cpuid assembly on both, which seems to be there since forever. + __get_cpuid[_count]() only does a bunch of bounds checking on top of + that anyway and if we didn't blow up on the assert above I HOPE we + should be fine to check for extended features level zero. For higher + levels (to check very recent AVX512 features) probably not but I + assume those would need very recent compilers anyway so when that + happens we can just pretend those don't exist on GCC 7.0 and + older. */ + #else + __cpuid_count(7, 0, cpuid.e.ax, cpuid.e.bx, cpuid.e.cx, cpuid.e.dx); + #endif + if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; + if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; + } - /* https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features */ + /* And now the LZCNT bit, finally + https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits */ #ifdef CORRADE_TARGET_MSVC - __cpuidex(cpuid.data, 7, 0); - /* __get_cpuid_count() is only since GCC 7.1 and Clang 5.0: - https://github.com/gcc-mirror/gcc/commit/2488ebe5ef1788616c2fbc61e05af09f0749ebbe - https://github.com/llvm/llvm-project/commit/f6e8408a116405d0cc25b506e8c5b60ab4ab7bcd - Furthermore, Clang 5.0 is 9.3 on Apple, 9.2 has only Clang 4: - https://en.wikipedia.org/wiki/Xcode#Toolchain_versions */ - #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 701) || (defined(CORRADE_TARGET_CLANG) && ((!defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__ >= 5) || (defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__*100 + __clang_minor__ >= 903))) - CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid_count(7, 0, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); - /* Looking at the above commits, on older GCC, __get_cpuid(7) seems to do - what __get_cpuid_count(7, 0) does on new GCC, however that's only since - 6.3: - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 - Older Clang doesn't do any of that, and thus to avoid a combinatorial - explosion of test scenarios, we'll just directly use the __cpuid - assembly on both, which seems to be there since forever. - __get_cpuid[_count]() only does a bunch of bounds checking on top of - that anyway and if we didn't blow up on the assert above I HOPE we - should be fine to check for extended features level zero. For higher - levels (to check very recent AVX512 features) probably not but I assume - those would need very recent compilers anyway so when that happens we - can just pretend those don't exist on GCC 7.0 and older. */ - #else - __cpuid_count(7, 0, cpuid.e.ax, cpuid.e.bx, cpuid.e.cx, cpuid.e.dx); + __cpuid(cpuid.data, 0x80000001); + #elif defined(CORRADE_TARGET_GCC) + CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid(0x80000001, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); #endif - if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; - if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 5)) out |= TypeTraits::Index; return Features{out}; @@ -258,6 +280,12 @@ Utility::Debug& operator<<(Utility::Debug& debug, const Features value) { _c(Avx) _c(Avx2) _c(Avx512f) + /* Print the extras at the end so the base instruction set is always first + even in case of Cpu::Default, where it's just one */ + _c(Popcnt) + _c(Lzcnt) + _c(AvxF16c) + _c(AvxFma) #elif defined(CORRADE_TARGET_ARM) _c(Neon) _c(NeonFma) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 2e8e1a57b..28d9d75c1 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -125,7 +125,7 @@ overload candidate for the set of CPU instruction features enabled at compile @m_class{m-block m-warning} -@par SSE3, SSSE3 and SSE4.1/SSE4.2 on MSVC +@par SSE3, SSSE3, SSE4.1/SSE4.2, POPCNT, LZCNT, AVX F16C and AVX FMA on MSVC A special case worth mentioning are SSE3 and newer instructions on Windows. MSVC only provides a very coarse `/arch:SSE2`, `/arch:AVX` and `/arch:AVX2` for either @ref Sse2, @ref Avx or @ref Avx2, but nothing in between. That @@ -152,7 +152,70 @@ returned @ref Cpu::Features against particular tags to decide which variant to @snippet Corrade.cpp Cpu-usage-runtime-manual-dispatch -@subsection Cpu-usage-target-attributes Enabling instruction sets for particular functions +@section Cpu-usage-extra Usage with extra instruction sets + +Besides the base instruction set, which on x86 is @ref Sse2 through +@ref Avx512f, with each tag being a superset of the previous one, there are +* *extra* instruction sets such as @ref Popcnt or @ref AvxFma. Basic +compile-time detection for these is still straightforward, only now using +@ref Default instead of @link DefaultBase @endlink: + +@snippet Corrade-cpp17.cpp Cpu-usage-extra-compile-time + +The process of defining and dispatching to function variants that include extra +instruction sets gets moderately more complex, however. As shown on the diagram +below, those are instruction sets that neither fit into the hierarchy nor are +unambiguously included in a later instruction set. For example, some CPUs are +known to have @ref Avx and just @ref AvxFma, some @ref Avx and just +@ref AvxF16c and there are even CPUs with @ref Avx2 but no @ref AvxFma. + +@dotfile cpu.dot + +While there's no possibility of having a total ordering between all possible +combinations for dispatching, the following approach is chosen: + +1. The base instruction set has the main priority. For example, if both an + @ref Avx2 and a @ref Sse2 variant are viable candidates, the @ref Avx2 + variant gets picked, even if the @ref Sse2 variant uses extra + instruction sets that the @ref Avx2 doesn't. +2. After that, the variant with the most extra instruction sets is chosen. For + example, an @ref Avx + @ref AvxFma variant is chosen over plain @ref Avx. + +On the declaration side, the desired base instruction set gets ORed with as +many extra instruction sets as needed, and then wrapped in a +@ref CORRADE_CPU_DECLARE() macro. For example, a lookup algorithm may have a +@ref Sse41 implementation which however also relies on @ref Popcnt and +@ref Lzcnt, and a fallback @ref Sse2 implementation that uses neither: + +@snippet Corrade.cpp Cpu-usage-extra-declare + +And a concrete overload gets picked at compile-time by passing a desired +combination of CPU tags as well --- or @ref Default for the set of features +enabled at compile time --- this time wrapped in a @ref CORRADE_CPU_SELECT(): + +@snippet Corrade.cpp Cpu-usage-extra-compile-time-call + + + +@m_class{m-block m-success} + +@par Resolving overload ambiguity + Because the best overload is picked based on the count of extra instruction + sets used, it may happen that two different variants get assigned the same + priority, causing an ambiguity. For example, the two variants below would + be abiguous for a CPU with @ref Sse41 and both @ref Popcnt and @ref Lzcnt + present: +@par + @snippet Corrade.cpp Cpu-usage-extra-ambiguity +@par + It's not desirable for this library to arbitrarily decide which instruction + set should be preferred --- only the implementation itself can know that. + Thus, to resolve such potential conflict, provide an overload with both + extra tags and delegate from there: +@par + @snippet Corrade.cpp Cpu-usage-extra-ambiguity-resolve + +@section Cpu-usage-target-attributes Enabling instruction sets for particular functions On GCC and Clang, a machine target has to be enabled in order to use a particular CPU instruction set or its intrinsics. While it's possible to do @@ -186,9 +249,10 @@ headers you won't use anyway. In comparison, using the @ref CORRADE_TARGET_SSE2 etc. macros would only make the variant available if the whole compilation unit has a corresponding `-m` or `/arch:` option passed to the compiler. -Definitions of the above functions would then look like below with the target -attributes enabled. The scalar variant has no target-specific annotation and -will be present as a fallback in all cases. +Definitions of the `lookup()` function variants from above would then look like +below with the target attributes added. The extra instruction sets get +explicitly enabled as well, in contrast a scalar variant would have no +target-specific annotations at all. @snippet Corrade.cpp Cpu-usage-target-attributes */ @@ -230,6 +294,8 @@ namespace Implementation { using {} */ struct InitT {}; constexpr InitT Init{}; + + enum: unsigned int { ExtraTagBitOffset = 16 }; } /** @@ -324,6 +390,34 @@ struct Sse42T: Sse41T { #endif }; +/** +@brief POPCNT tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Popcnt tag for more information. +@see @ref tag(), @ref features() +*/ +struct PopcntT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit PopcntT(Implementation::InitT) {} + #endif +}; + +/** +@brief LZCNT tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Lzcnt tag for more information. +@see @ref tag(), @ref features() +*/ +struct LzcntT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit LzcntT(Implementation::InitT) {} + #endif +}; + /** @brief AVX tag type @@ -338,6 +432,34 @@ struct AvxT: Sse42T { #endif }; +/** +@brief AVX F16C tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref AvxF16c tag for more information. +@see @ref tag(), @ref features() +*/ +struct AvxF16cT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit AvxF16cT(Implementation::InitT) {} + #endif +}; + +/** +@brief AVX FMA tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref AvxFma tag for more information. +@see @ref tag(), @ref features() +*/ +struct AvxFmaT { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit AvxFmaT(Implementation::InitT) {} + #endif +}; + /** @brief AVX2 tag type @@ -367,6 +489,7 @@ struct Avx512fT: Avx2T { }; #ifndef DOXYGEN_GENERATING_OUTPUT +/* Features earlier in the hierarchy should have lower bits set */ template<> struct TypeTraits { enum: unsigned int { Index = 1 << 0 }; static const char* name() { return "Sse2"; } @@ -399,6 +522,24 @@ template<> struct TypeTraits { enum: unsigned int { Index = 1 << 7 }; static const char* name() { return "Avx512f"; } }; + +/* The total bit range should not be larger than ExtraTagCount */ +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (0 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "Popcnt"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (1 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "Lzcnt"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (2 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "AvxF16c"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (3 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "AvxFma"; } +}; #endif #endif @@ -554,6 +695,33 @@ implied by @ref Avx. */ constexpr Sse42T Sse42{Implementation::Init}; +/** +@brief POPCNT tag + +[POPCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions. Available only on @ref CORRADE_TARGET_X86 "x86". This instruction +set is treated as an *extra*, i.e. is neither a superset of nor implied by any +other instruction set. See @ref Cpu-usage-extra for more information. +@see @ref Lzcnt, @ref CORRADE_TARGET_POPCNT, @ref CORRADE_ENABLE_POPCNT +*/ +constexpr PopcntT Popcnt{Implementation::Init}; + +/** +@brief LZCNT tag + +[LZCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions. Available only on @ref CORRADE_TARGET_X86 "x86". This instruction +set is treated as an *extra*, i.e. is neither a superset of nor implied by any +other instruction set. See @ref Cpu-usage-extra for more information. + +Note that this instruction has encoding compatible with an earlier `BSR` +instruction which has a slightly different behavior. To avoid wrong results if +it isn't available, prefer to always detect its presence with +@ref runtimeFeatures() instead of a compile-time check. +@see @ref Popcnt, @ref CORRADE_TARGET_LZCNT, @ref CORRADE_ENABLE_LZCNT +*/ +constexpr LzcntT Lzcnt{Implementation::Init}; + /** @brief AVX tag @@ -564,6 +732,28 @@ implied by @ref Avx2. */ constexpr AvxT Avx{Implementation::Init}; +/** +@brief AVX F16C tag + +[F16C](https://en.wikipedia.org/wiki/F16C) instructions. Available only on +@ref CORRADE_TARGET_X86 "x86". This instruction set is treated as an *extra*, +i.e. is neither a superset of nor implied by any other instruction set. See +@ref Cpu-usage-extra for more information. +@see @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_ENABLE_AVX_F16C +*/ +constexpr AvxF16cT AvxF16c{Implementation::Init}; + +/** +@brief AVX FMA tag + +[FMA3 instruction set](https://en.wikipedia.org/wiki/FMA_instruction_set). +Available only on @ref CORRADE_TARGET_X86 "x86". This instruction set is +treated as an *extra*, i.e. is neither a superset of nor implied by any other +instruction set. See @ref Cpu-usage-extra for more information. +@see @ref CORRADE_TARGET_AVX_FMA, @ref CORRADE_ENABLE_AVX_FMA +*/ +constexpr AvxFmaT AvxFma{Implementation::Init}; + /** @brief AVX2 tag @@ -627,6 +817,139 @@ on @ref CORRADE_TARGET_WASM "WebAssembly". Superset of @ref Scalar. constexpr Simd128T Simd128{Implementation::Init}; #endif +namespace Implementation { + +template struct Priority: Priority {}; +template<> struct Priority<0> {}; + +/* Count of "extra" tags that are not in the hierarchy. Should not be larger + than strictly necessary as it deepens inheritance hierarchy when picking + best overload candidate. */ +enum: unsigned int { + BaseTagMask = (1 << ExtraTagBitOffset) - 1, + ExtraTagMask = 0xffffffffu & ~BaseTagMask, + #ifdef CORRADE_TARGET_X86 + ExtraTagCount = 4, + #else + ExtraTagCount = 0, + #endif +}; + +/* Holds a compile-time combination of tags. Kept private, since it isn't + really directly needed in user code and it would only lead to confusion. */ +template struct Tags { + enum: unsigned int { Value = value }; + + /* Empty initialization. Should not be needed by public code. */ + constexpr explicit Tags(InitT) {} + + /* Conversion from other tag combination, allowed only if the other is + not a subset */ + template constexpr Tags(Tags, typename std::enable_if< + /* There should be at most one base tag set in both, if there's none + then it's Cpu::Scalar */ + !((value & BaseTagMask) & ((value & BaseTagMask) - 1)) && + !((otherValue & BaseTagMask) & ((otherValue & BaseTagMask) - 1)) && + /* The other base tag should be the same or derived (i.e, having same + or larger value) */ + (otherValue & BaseTagMask) >= (value & BaseTagMask) && + /* The other extra bits should be a superset of this */ + ((otherValue & value) & ExtraTagMask) == (value & ExtraTagMask) + >::type* = {}) {} + + /* Conversion from a single tag, allowed only if we're a single bit and + the other is not a subset */ + template constexpr Tags(T, typename std::enable_if< + /* There should be at most one base tag set in this one, the other + satisfies that implicitly as it's constrained by TypeTraits */ + !((value & BaseTagMask) & ((value & BaseTagMask) - 1)) && + /* The other base tag should be the same or derived (i.e, having same + or larger value) */ + (TypeTraits::Index & BaseTagMask) >= (value & BaseTagMask) && + /* The other extra bits should be a superset of this. Since a single + tag can be only one bit, this condition gets satisfied only either + if we're Cpu::Scalar or if we have no extra bits. */ + ((TypeTraits::Index & value) & ExtraTagMask) == (value & ExtraTagMask) + >::type* = {}) {} + + /* A subset of operators on Features, excluding the assignment ones -- + since they modify the type, they make no sense here */ + template constexpr Tags::Index> operator|(U) const { + return Tags::Index>{Init}; + } + template constexpr Tags::Index> operator&(U) const { + return Tags::Index>{Init}; + } + template constexpr Tags::Index> operator^(U) const { + return Tags::Index>{Init}; + } + constexpr Tags<~value> operator~() const { + return Tags<~value>{Init}; + } + constexpr explicit operator bool() const { return value; } + constexpr operator unsigned int() const { return value; } +}; + +template constexpr Tags::Index> tags(T) { + return Tags::Index>{Init}; +} +template constexpr Tags tags(Tags tags) { + return tags; +} + +/* Base-2 log, "plus one" (returns 0 for A == 0). For a base tag, which is + always just one bit, returns position of that bit. Used for calculating + distance between two tags in order to calculate overload priority. But since + there's also the Scalar tag, which is 0, it's plus one. */ +template struct BitIndex { + enum: unsigned short { Value = 1 + BitIndex<(A >> 1)>::Value }; +}; +template<> struct BitIndex<0> { + enum: unsigned short { Value = 0 }; +}; + +/* Popcount, but constexpr. No, not related to Cpu::Popcnt, in any way. Used + for calculating a difference between two extra tag sets in order to + calculate overload priority. */ +template struct BitCount { + /* https://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation; + lol I can do that without having to recurse */ + enum: unsigned short { + Bits1 = 0x5555, /* 0b0101010101010101 */ + Bits2 = 0x3333, /* 0b0011001100110011 */ + Bits4 = 0x0f0f, /* 0b0000111100001111 */ + Bits8 = 0x00ff, /* 0b0000000011111111 */ + + B0 = (A >> 0) & Bits1, + B1 = (A >> 1) & Bits1, + C = B0 + B1, + D0 = (C >> 0) & Bits2, + D2 = (C >> 2) & Bits2, + E = D0 + D2, + F0 = (E >> 0) & Bits4, + F4 = (E >> 4) & Bits4, + G = F0 + F4, + H0 = (G >> 0) & Bits8, + H8 = (G >> 8) & Bits8, + Value = H0 + H8 + }; +}; + +/* Calculates an absolute priority index for given tag, which is either a + base-2 log "plus one" of its index if it's a base tag, or is 1 if it's an + extra tag */ +template Priority::Index & ExtraTagMask ? 1 : BitIndex::Index & BaseTagMask>::Value*(ExtraTagCount + 1)> constexpr priority(T) { + return {}; +} +template Priority::Value*(ExtraTagCount + 1) + BitCount<((value & ExtraTagMask) >> ExtraTagBitOffset)>::Value> constexpr priority(Tags) { + static_assert(!((value & BaseTagMask) & ((value & BaseTagMask) - 1)), "more than one base tag used"); + /* GCC 4.8 loudly complains about enum comparison if I don't cast, sigh */ + static_assert(((value & ExtraTagMask) >> ExtraTagBitOffset) < (1 << static_cast(ExtraTagCount)), "extra tag out of expected bounds"); + return {}; +} + +} + /** @brief Default base tag type @@ -674,6 +997,35 @@ typedef #endif DefaultBaseT; +/** +@brief Default extra tag type + +See the @ref DefaultExtra tag for more information. +*/ +typedef Implementation::Tags< + #ifdef CORRADE_TARGET_X86 + #ifdef CORRADE_TARGET_POPCNT + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_LZCNT + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX_FMA + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX_F16C + TypeTraits::Index| + #endif + #endif + 0> DefaultExtraT; + +/** +@brief Default tag type + +See the @ref Default tag for more information. +*/ +typedef Implementation::Tags::Index|DefaultExtraT::Value> DefaultT; + /** @brief Default base tag @@ -702,12 +1054,46 @@ On @ref CORRADE_TARGET_WASM it's one of these: - @ref Simd128 if @ref CORRADE_TARGET_SIMD128 is defined - @ref Scalar otherwise -See also @ref compiledFeatures(), which returns a *combination* of these tags -instead of just the highest available, and @ref runtimeFeatures() which is +In addition to the above, @ref DefaultExtra contains a combination of extra +instruction sets available together with the base instruction set, and +@ref Default is a combination of both. See also @ref compiledFeatures() which +returns a *combination* of base tags instead of just the highest available, +together with the extra instruction sets, and @ref runtimeFeatures() which is capable of detecting the available CPU feature set at runtime. */ constexpr DefaultBaseT DefaultBase{Implementation::Init}; +/** +@brief Default extra tags + +Instruction sets available in addition to @ref DefaultBase on +given architecture with current compiler flags. On @ref CORRADE_TARGET_X86 it's +a combination of these: + +- @ref Popcnt if @ref CORRADE_TARGET_POPCNT is defined +- @ref Lzcnt if @ref CORRADE_TARGET_LZCNT is defined +- @ref AvxFma if @ref CORRADE_TARGET_AVX_FMA is defined +- @ref AvxF16c if @ref CORRADE_TARGET_AVX_F16C is defined + +No extra instruction sets are currently defined for @ref CORRADE_TARGET_ARM or +@ref CORRADE_TARGET_WASM. + +In addition to the above, @ref Default is a combination of both +@ref DefaultBase and the extra instruction sets. See also +@ref compiledFeatures() which returns these together with a combination of all +base instruction sets available, and @ref runtimeFeatures() which is capable of +detecting the available CPU feature set at runtime. +*/ +constexpr DefaultExtraT DefaultExtra{Implementation::Init}; + +/** +@brief Default tags + +A combination of @ref DefaultBase and @ref DefaultExtra, see their +documentation for more information. +*/ +constexpr DefaultT Default{Implementation::Init}; + /** @brief Tag for a tag type @@ -752,7 +1138,17 @@ class Features { * * @see @ref features() */ - template::Index)> constexpr /*implicit*/ Features(T) noexcept: _data{TypeTraits::Index} {} + template::Index)> constexpr /*implicit*/ Features(T) noexcept: _data{TypeTraits::Index} { + /* GCC 4.8 loudly complains about enum comparison if I don't cast, sigh */ + static_assert(((TypeTraits::Index & Implementation::ExtraTagMask) >> Implementation::ExtraTagBitOffset) < (1 << static_cast(Implementation::ExtraTagCount)), + "extra tag out of expected bounds"); + } + + #ifndef DOXYGEN_GENERATING_OUTPUT + /* The compile-time Tags<> is an implementation detail, don't show that + in the docs */ + template constexpr /*implicit*/ Features(Implementation::Tags) noexcept: _data{value} {} + #endif /** @brief Equality comparison */ constexpr bool operator==(Features other) const { @@ -912,6 +1308,18 @@ template::Index)> constexpr Features ope return b | a; } +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Compared to the above, this produces a type that encodes the value instead + of Features. Has to be in the same namespace as the tags, but since Tags<> + is an implementation detail, this is hidden from plain sight as well. */ +template constexpr Implementation::Tags::Index | TypeTraits::Index> operator|(T, U) { + return Implementation::Tags::Index | TypeTraits::Index>{Implementation::Init}; +} +template constexpr Implementation::Tags::Index | value> operator|(T, Implementation::Tags) { + return Implementation::Tags::Index | value>{Implementation::Init}; +} +#endif + /** @relates Features @brief Intersection of two feature sets @@ -921,6 +1329,18 @@ template::Index)> constexpr Features ope return b & a; } +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Compared to the above, this produces a type that encodes the value instead + of Features. Has to be in the same namespace as the tags, but since Tags<> + is an implementation detail, this is hidden from plain sight as well. */ +template constexpr Implementation::Tags::Index & TypeTraits::Index> operator&(T, U) { + return Implementation::Tags::Index & TypeTraits::Index>{Implementation::Init}; +} +template constexpr Implementation::Tags::Index & value> operator&(T, Implementation::Tags) { + return Implementation::Tags::Index & value>{Implementation::Init}; +} +#endif + /** @relates Features @brief XOR of two feature sets @@ -930,14 +1350,32 @@ template::Index)> constexpr Features ope return b ^ a; } +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Compared to the above, this produces a type that encodes the value instead + of Features. Has to be in the same namespace as the tags, but since Tags<> + is an implementation detail, this is hidden from plain sight as well. */ +template constexpr Implementation::Tags::Index ^ TypeTraits::Index> operator^(T, U) { + return Implementation::Tags::Index ^ TypeTraits::Index>{Implementation::Init}; +} +template constexpr Implementation::Tags::Index ^ value> operator^(T, Implementation::Tags) { + return Implementation::Tags::Index ^ value>{Implementation::Init}; +} +#endif + /** @relates Features @brief Feature set complement Same as @ref Features::operator~(). */ -template::Index)> constexpr Features operator~(T a) { - return ~Features(a); +#ifdef DOXYGEN_GENERATING_OUTPUT +template constexpr Features operator~(T a); +#else +/* To avoid confusion, to the doc use it's shown that the operator produces a + Features, but in fact it's a type with a compile-time-encoded value */ +template constexpr Implementation::Tags<~TypeTraits::Index> operator~(T) { + return Implementation::Tags<~TypeTraits::Index>{Implementation::Init}; } +#endif /** @debugoperator{Features} */ CORRADE_UTILITY_EXPORT Utility::Debug& operator<<(Utility::Debug& debug, Features value); @@ -949,12 +1387,19 @@ template::Index)> inline Utility::Debug& return operator<<(debug, Features{value}); } +namespace Implementation { + template inline Utility::Debug& operator<<(Utility::Debug& debug, Tags value) { + return operator<<(debug, Features{value}); + } +} + /** @brief CPU instruction sets enabled at compile time On @ref CORRADE_TARGET_X86 "x86" returns a combination of @ref Sse2, @ref Sse3, -@ref Ssse3, @ref Sse41, @ref Sse42, @ref Avx, @ref Avx2 and @ref Avx512f based -on what all @ref CORRADE_TARGET_SSE2 etc. preprocessor variables are defined. +@ref Ssse3, @ref Sse41, @ref Sse42, @ref Popcnt, @ref Lzcnt, @ref Avx, +@ref AvxF16c, @ref AvxFma, @ref Avx2 and @ref Avx512f based on what all +@ref CORRADE_TARGET_SSE2 etc. preprocessor variables are defined. On @ref CORRADE_TARGET_ARM "ARM", returns a combination of @ref Neon, @ref NeonFma and @ref NeonFp16 based on what all @ref CORRADE_TARGET_NEON etc. @@ -966,7 +1411,7 @@ whether the @ref CORRADE_TARGET_SIMD128 preprocessor variable is defined. On other platforms or if no known CPU instruction set is enabled, the returned value is equal to @ref Scalar, which in turn is equivalent to empty (or default-constructed) @ref Features. -@see @ref DefaultBase +@see @ref DefaultBase, @ref DefaultExtra, @ref Default */ constexpr Features compiledFeatures() { return Features{ @@ -986,9 +1431,21 @@ constexpr Features compiledFeatures() { #ifdef CORRADE_TARGET_SSE42 TypeTraits::Index| #endif + #ifdef CORRADE_TARGET_POPCNT + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_LZCNT + TypeTraits::Index| + #endif #ifdef CORRADE_TARGET_AVX TypeTraits::Index| #endif + #ifdef CORRADE_TARGET_AVX_FMA + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_AVX_F16C + TypeTraits::Index| + #endif #ifdef CORRADE_TARGET_AVX2 TypeTraits::Index| #endif @@ -1017,11 +1474,11 @@ constexpr Features compiledFeatures() { On @ref CORRADE_TARGET_X86 "x86" and GCC, Clang or MSVC uses the [CPUID](https://en.wikipedia.org/wiki/CPUID) builtin to check for the -@ref Sse2, @ref Sse3, @ref Ssse3, @ref Sse41, @ref Sse42, @ref Avx, @ref Avx2 -and @ref Avx512f runtime features. @ref Avx needs OS support as well, if it's -not present, no following flags are checked either. On compilers other than -GCC, Clang and MSVC the function is @cpp constexpr @ce and delegates into -@ref compiledFeatures(). +@ref Sse2, @ref Sse3, @ref Ssse3, @ref Sse41, @ref Sse42, @ref Popcnt, +@ref Lzcnt, @ref Avx, @ref AvxF16c, @ref AvxFma, @ref Avx2 and @ref Avx512f +runtime features. @ref Avx needs OS support as well, if it's not present, no +following flags are checked either. On compilers other than GCC, Clang and MSVC +the function is @cpp constexpr @ce and delegates into @ref compiledFeatures(). On @ref CORRADE_TARGET_ARM "ARM" and Linux or Android API level 18+ uses @m_class{m-doc-external} [getauxval()](https://man.archlinux.org/man/getauxval.3) @@ -1039,8 +1496,7 @@ into @ref compiledFeatures(). On other platforms or if no known CPU instruction set is detected, the returned value is equal to @ref Scalar, which in turn is equivalent to empty (or default-constructed) @ref Features. - -@see @ref DefaultBase +@see @ref DefaultBase, @ref DefaultExtra, @ref Default */ #if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(DOXYGEN_GENERATING_OUTPUT) CORRADE_UTILITY_EXPORT Features runtimeFeatures(); @@ -1048,6 +1504,37 @@ CORRADE_UTILITY_EXPORT Features runtimeFeatures(); constexpr Features runtimeFeatures() { return compiledFeatures(); } #endif +/** +@brief Declare a CPU tag for a compile-time dispatch +@m_since_latest + +Meant to be used to declare a function overload that uses given combination of +CPU instruction sets. The @ref CORRADE_CPU_SELECT() macro is a counterpart used +to select among overloads declared with this macro. See @ref Cpu-usage-extra +for more information and usage example. + +Internally, this macro expands to two function parameter declarations separated +by a comma, one that ensures only an overload matching the desired instruction +sets get picked, and one that assigns an absolute priority to this overload. +*/ +#define CORRADE_CPU_DECLARE(tag) decltype(Corrade::Cpu::Implementation::tags(tag)), decltype(Corrade::Cpu::Implementation::priority(tag)) + +/** +@brief Select a CPU tag for a compile-time dispatch +@m_since_latest + +Meant to be used to select among function overloads declared with +@ref CORRADE_CPU_DECLARE() that best matches given combination of CPU +instruction sets. See @ref Cpu-usage-extra for more information and usage +example. + +Internally, this macro expands to two function parameter values separated by a +comma, one that contains the desired instruction sets to filter the overloads +against and another that converts the sets to an absolute priority to pick the +best viable overload. +*/ +#define CORRADE_CPU_SELECT(tag) tag, Corrade::Cpu::Implementation::priority(tag) + #if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) /** @brief Enable SSE2 for given function @@ -1244,6 +1731,96 @@ Superset of @ref CORRADE_ENABLE_SSE41, implied by @ref CORRADE_ENABLE_AVX. See #define CORRADE_ENABLE_SSE42 #endif +/** +@brief Enable POPCNT for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang and @ref CORRADE_TARGET_CLANG_CL "clang-cl" +expands to @cpp __attribute__((__target__("popcnt"))) @ce, allowing use of the +[POPCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions inside a function annotated with this macro without having to +specify `-mpopcnt` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Not +defined on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_POPCNT is defined (meaning POCNT is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Neither a superset nor implied by any other `CORRADE_ENABLE_*` macro, so you +may need to specify it together with others. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsSse4.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Popcnt} +*/ +#if defined(CORRADE_TARGET_POPCNT) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_POPCNT +#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_POPCNT __attribute__((__target__("popcnt"))) +#elif defined(CORRADE_TARGET_MSVC) +#define CORRADE_ENABLE_POPCNT +#endif + +/** +@brief Enable LZCNT for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("lzcnt"))) @ce, allowing use of the +[LZCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) +instructions inside a function annotated with this macro without having to +specify `-mlzcnt` for the whole compilation unit. On x86 MSVC expands to +nothing, as the compiler doesn't restrict use of intrinsics in any way. Unlike +the SSE variants and POPCNT this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined +on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_LZCNT is defined (meaning LZCNT is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Neither a superset nor implied by any other `CORRADE_ENABLE_*` macro, so you +may need to specify it together with others. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Lzcnt} +*/ +#if defined(CORRADE_TARGET_LZCNT) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_LZCNT +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +/* GCC 4.8 needs also -mabm. This option is not needed since 4.9 and is also + unrecognized on Clang. */ +#if defined(CORRADE_TARGET_CLANG) || __GNUC__*100 + __GNUC_MINOR__ >= 409 +#define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt"))) +#else +#define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt,abm"))) +#endif +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __LZCNT__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_LZCNT +#endif + /** @brief Enable AVX for given function @m_since_latest @@ -1291,6 +1868,102 @@ Superset of @ref CORRADE_ENABLE_SSE42, implied by @ref CORRADE_ENABLE_AVX2. See #define CORRADE_ENABLE_AVX #endif +/** +@brief Enable AVX F16C for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("f16c"))) @ce, allowing use of +[F16C](https://en.wikipedia.org/wiki/F16C) instructions inside a function +annotated with this macro without having to specify `-mf16c` for the whole +compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Unlike the SSE variants this macro is +not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined +on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX_F16C is present (meaning AVX F16C +is enabled for the whole compilation unit), this macro is defined as empty on +all compilers. + +Superset of @ref CORRADE_ENABLE_AVX on both GCC and Clang. However not +portably implied by any other `CORRADE_ENABLE_*` macro so you may need to +specify it together with others. See @ref Cpu-usage-target-attributes for more +information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::AvxF16c} +*/ +#if defined(CORRADE_TARGET_AVX_F16C) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX_F16C +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +/* The -mf16c option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on + both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ +#define CORRADE_ENABLE_AVX_F16C __attribute__((__target__("f16c"))) +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __F16C__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_AVX_F16C +#endif + +/** +@brief Enable AVX FMA for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC and Clang expands to +@cpp __attribute__((__target__("fma"))) @ce, allowing use of +[FMA](https://en.wikipedia.org/wiki/FMA_instruction_set) instructions inside a +function annotated with this macro without having to specify `-mfma` for the +whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't +restrict use of intrinsics in any way. Unlike the SSE variants this macro is +not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined +on other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_AVX_FMA is present (meaning AVX with +FMA is enabled for the whole compilation unit), this macro is defined as empty +on all compilers. + +Superset of @ref CORRADE_ENABLE_AVX on both GCC and Clang. However not +portably implied by any other `CORRADE_ENABLE_*` macro so you may need to +specify it together with others. See @ref Cpu-usage-target-attributes for more +information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::AvxFma} +*/ +#if defined(CORRADE_TARGET_AVX_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_AVX_FMA +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +/* The -mfma option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on + both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ +#define CORRADE_ENABLE_AVX_FMA __attribute__((__target__("fma"))) +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __FMA__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_AVX_FMA +#endif + /** @brief Enable AVX2 for given function @m_since_latest diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index c332c2fe6..cf99b0287 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -84,7 +84,18 @@ struct CpuTest: TestSuite::Tester { void detectDefault(); void detect(); + void bitIndex(); + void bitCount(); + void priority(); + void tagDispatch(); + void tagDispatchExtraExact(); + void tagDispatchExtraUnusedExtra(); + void tagDispatchExtraFallbackExtra(); + void tagDispatchExtraFallbackBase(); + void tagDispatchExtraFallbackBoth(); + void tagDispatchExtraPriority(); + template void enableMacros(); void debug(); @@ -122,14 +133,29 @@ CpuTest::CpuTest() { addInstancedTests({&CpuTest::detect}, Containers::arraySize(DetectData)); - addTests({&CpuTest::tagDispatch, + addTests({&CpuTest::bitIndex, + &CpuTest::bitCount, + &CpuTest::priority, + + &CpuTest::tagDispatch, + &CpuTest::tagDispatchExtraExact, + &CpuTest::tagDispatchExtraUnusedExtra, + &CpuTest::tagDispatchExtraFallbackBase, + &CpuTest::tagDispatchExtraFallbackExtra, + &CpuTest::tagDispatchExtraFallbackBoth, + &CpuTest::tagDispatchExtraPriority, + #ifdef CORRADE_TARGET_X86 &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, &CpuTest::enableMacros, + &CpuTest::enableMacros, + &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, #elif defined(CORRADE_TARGET_ARM) @@ -157,6 +183,8 @@ void CpuTest::tagNoDefaultConstructor() { CORRADE_VERIFY(!std::is_default_constructible::value); CORRADE_VERIFY(!std::is_default_constructible::value); CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); + CORRADE_VERIFY(!std::is_default_constructible::value); CORRADE_VERIFY(!std::is_default_constructible::value); CORRADE_VERIFY(!std::is_default_constructible::value); #elif defined(CORRADE_TARGET_ARM) @@ -178,6 +206,8 @@ void CpuTest::tagInlineDefinition() { CORRADE_VERIFY(std::is_same::value); CORRADE_VERIFY(std::is_same::value); CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); + CORRADE_VERIFY(std::is_same::value); CORRADE_VERIFY(std::is_same::value); CORRADE_VERIFY(std::is_same::value); #elif defined(CORRADE_TARGET_ARM) @@ -287,6 +317,10 @@ void CpuTest::featuresConstructTemplate() { void CpuTest::featuresOperatorOr() { #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ Cpu::Features features = Cpu::Sse3|Cpu::Sse2; CORRADE_COMPARE(std::uint32_t(features), 3); @@ -302,6 +336,17 @@ void CpuTest::featuresOperatorOr() { CORRADE_COMPARE(std::uint32_t(cFeatures), 3); CORRADE_COMPARE(std::uint32_t(cFeatures1), 7); CORRADE_COMPARE(std::uint32_t(cFeatures2), 7); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags = Cpu::Sse3|Cpu::Sse2; + constexpr auto cTags1 = cTags|Cpu::Ssse3; + constexpr auto cTags2 = Cpu::Ssse3|cTags; + CORRADE_COMPARE(std::uint32_t(cTags), 3); + CORRADE_COMPARE(std::uint32_t(cTags1), 7); + CORRADE_COMPARE(std::uint32_t(cTags2), 7); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(std::is_same::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -309,6 +354,10 @@ void CpuTest::featuresOperatorOr() { void CpuTest::featuresOperatorAnd() { #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 & Cpu::Sse2), 0); Cpu::Features features = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; @@ -328,6 +377,16 @@ void CpuTest::featuresOperatorAnd() { constexpr Cpu::Features cFeatures2 = Cpu::Sse41 & cFeatures; CORRADE_COMPARE(std::uint32_t(cFeatures1), 8); CORRADE_COMPARE(std::uint32_t(cFeatures2), 8); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr auto cTags1 = cTags & Cpu::Sse41; + constexpr auto cTags2 = Cpu::Sse41 & cTags; + CORRADE_COMPARE(std::uint32_t(cTags1), 8); + CORRADE_COMPARE(std::uint32_t(cTags2), 8); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(std::is_same::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -335,6 +394,10 @@ void CpuTest::featuresOperatorAnd() { void CpuTest::featuresOperatorXor() { #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 ^ Cpu::Sse3), 0); CORRADE_COMPARE(std::uint32_t(Cpu::Sse3 ^ Cpu::Sse2), 3); @@ -355,6 +418,16 @@ void CpuTest::featuresOperatorXor() { constexpr Cpu::Features cFeatures2 = Cpu::Sse2 ^ cFeatures; CORRADE_COMPARE(std::uint32_t(cFeatures1), 10); CORRADE_COMPARE(std::uint32_t(cFeatures2), 10); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr auto cTags1 = cTags ^ Cpu::Sse2; + constexpr auto cTags2 = Cpu::Sse2 ^ cTags; + CORRADE_COMPARE(std::uint32_t(cTags1), 10); + CORRADE_COMPARE(std::uint32_t(cTags2), 10); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(std::is_same::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -365,6 +438,11 @@ void CpuTest::featuresOperatorBoolScalar() { constexpr bool cFeatures = !!Cpu::Features{Cpu::Scalar}; CORRADE_VERIFY(!cFeatures); + + /* Have to use an implementation detail to create the Tags type here. Which + is fine, people shouldn't need to do this directly. */ + constexpr bool cTags = !!Cpu::Implementation::tags(Cpu::Scalar); + CORRADE_VERIFY(!cTags); } void CpuTest::featuresOperatorBool() { @@ -378,6 +456,14 @@ void CpuTest::featuresOperatorBool() { constexpr bool cFeatures2 = !!(cFeatures & Cpu::Sse3); CORRADE_VERIFY(!cFeatures1); CORRADE_VERIFY(cFeatures2); + + /* Test also the compile-time operation */ + constexpr auto cTags = Cpu::Sse3|Cpu::Sse2; + constexpr bool cTags1 = !!(cTags & Cpu::Sse41); + constexpr bool cTags2 = !!(cTags & Cpu::Sse3); + CORRADE_VERIFY(!cTags1); + CORRADE_VERIFY(cTags2); + CORRADE_VERIFY(!std::is_same::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -385,6 +471,10 @@ void CpuTest::featuresOperatorBool() { void CpuTest::featuresOperatorInverse() { #ifdef CORRADE_TARGET_X86 + /* This is actually using the compile-time operation, producing Tags, + which is tested explicitly below, but that should be completely + transparent to the user and work as if it produced Features directly + instead of going through Tags first */ CORRADE_COMPARE(std::uint32_t(~Cpu::Scalar), 0xffffffffu); CORRADE_COMPARE(std::uint32_t(~(Cpu::Sse41|Cpu::Sse3)), 4294967285u); CORRADE_COMPARE(std::uint32_t(~Cpu::Sse41), 4294967287u); @@ -393,6 +483,15 @@ void CpuTest::featuresOperatorInverse() { constexpr Cpu::Features cFeatures2 = ~(Cpu::Sse41|Cpu::Sse3); CORRADE_COMPARE(std::uint32_t(cFeatures1), 0xffffffffu); CORRADE_COMPARE(std::uint32_t(cFeatures2), 4294967285u); + + /* Test also the compile-time operation, different values should be + different types but same values should be same types */ + constexpr auto cTags1 = ~Cpu::Scalar; + constexpr auto cTags2 = ~(Cpu::Sse41|Cpu::Sse3); + CORRADE_COMPARE(std::uint32_t(cTags1), 0xffffffffu); + CORRADE_COMPARE(std::uint32_t(cTags2), 4294967285u); + CORRADE_VERIFY(!std::is_same::value); + CORRADE_VERIFY(!std::is_same::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -427,6 +526,17 @@ void CpuTest::featuresCompare() { CORRADE_VERIFY(!cFeaturesNonEqual); CORRADE_VERIFY(cFeaturesLessEqual); CORRADE_VERIFY(cFeaturesGreaterEqual); + + constexpr auto cTags = Cpu::Sse41|Cpu::Sse2|Cpu::Sse3; + constexpr bool cTagsEqual = cTags == cTags; + constexpr bool cTagsNonEqual = cTags != cTags; + constexpr bool cTagsLessEqual = cTags <= cTags; + constexpr bool cTagsGreaterEqual = cTags >= cTags; + CORRADE_VERIFY(cTagsEqual); + CORRADE_VERIFY(!cTagsNonEqual); + CORRADE_VERIFY(cTagsLessEqual); + CORRADE_VERIFY(cTagsGreaterEqual); + CORRADE_VERIFY(!std::is_same::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -475,6 +585,57 @@ void CpuTest::detect() { #endif } +void CpuTest::bitIndex() { + CORRADE_COMPARE(Cpu::Implementation::BitIndex<0>::Value, 0); + CORRADE_COMPARE(Cpu::Implementation::BitIndex<1>::Value, 1); + CORRADE_COMPARE(Cpu::Implementation::BitIndex<(1 << 7)>::Value, 8); + CORRADE_COMPARE(Cpu::Implementation::BitIndex<(1 << 15)>::Value, 16); +} + +void CpuTest::bitCount() { + CORRADE_COMPARE(Cpu::Implementation::BitCount<0>::Value, 0); + CORRADE_COMPARE(Cpu::Implementation::BitCount<(1 << 7)>::Value, 1); + CORRADE_COMPARE(Cpu::Implementation::BitCount<12345>::Value, 6); + CORRADE_COMPARE(Cpu::Implementation::BitCount<65432>::Value, 11); + CORRADE_COMPARE(Cpu::Implementation::BitCount<0xffffu>::Value, 16); +} + +template unsigned int priorityValue(Cpu::Implementation::Priority) { + return i; +} + +void CpuTest::priority() { + #ifdef CORRADE_TARGET_X86 + /* Extra tag alone is always 1 */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::AvxFma)), 1); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Popcnt)), 1); + + /* More extra tags together is their count */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::AvxFma|Cpu::AvxF16c|Cpu::Lzcnt)), 3); + + /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus + zero, times the count of extra tags plus one */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Scalar)), 0); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Sse2)), 1*5); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2)), 7*5); + + /* Base tag + extra tags is a sum of the two */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), 7*5 + 2); + #elif defined(CORRADE_TARGET_ARM) + /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus + zero, times one as there are no extra tags */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Scalar)), 0); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Neon)), 1); + #elif defined(CORRADE_TARGET_WASM) + /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus + zero, times one as there are no extra tags */ + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Scalar)), 0); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Simd128)), 1); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + #if defined(CORRADE_TARGET_X86) || defined(CORRADE_TARGET_ARM) const char* dispatch(Cpu::ScalarT) { return "scalar"; } #ifdef CORRADE_TARGET_X86 @@ -511,6 +672,138 @@ void CpuTest::tagDispatch() { #endif } +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)) { return "AVX2+FMA+F16C"; } +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Popcnt)) { return "SSE4.2+POPCNT"; } +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Ssse3)) { return "SSSE3"; } +const char* dispatchExtraExact(CORRADE_CPU_DECLARE(Cpu::Popcnt|Cpu::Lzcnt)) { return "POPCNT+LZCNT"; } +#endif + +void CpuTest::tagDispatchExtraExact() { + #ifdef CORRADE_TARGET_X86 + /* For each there's an exact matching overload */ + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2+FMA+F16C"_s); + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2+POPCNT"_s); + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Ssse3)), "SSSE3"_s); + /* The base tag doesn't even need to be there */ + CORRADE_COMPARE(dispatchExtraExact(CORRADE_CPU_SELECT(Cpu::Popcnt|Cpu::Lzcnt)), "POPCNT+LZCNT"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Avx2)) { return "AVX2"; } +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Sse42)) { return "SSE4.2"; } +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Ssse3)) { return "SSSE3"; } +const char* dispatchExtraUnusedExtra(CORRADE_CPU_DECLARE(Cpu::Scalar)) { return "scalar"; } +#endif + +void CpuTest::tagDispatchExtraUnusedExtra() { + #ifdef CORRADE_TARGET_X86 + /* The extra tags get ignored, only the base one will be used */ + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2"_s); + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2"_s); + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Ssse3)), "SSSE3"_s); + /* The base tag doesn't even need to be there */ + CORRADE_COMPARE(dispatchExtraUnusedExtra(CORRADE_CPU_SELECT(Cpu::Popcnt|Cpu::Lzcnt)), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraFallbackExtra(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt)) { return "AVX2+POPCNT+LZCNT"; } +const char* dispatchExtraFallbackExtra(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Lzcnt)) { return "SSE4.2+LZCNT"; } +const char* dispatchExtraFallbackExtra(CORRADE_CPU_DECLARE(Cpu::Sse42)) { return "SSE4.2"; } +#endif + +void CpuTest::tagDispatchExtraFallbackExtra() { + #ifdef CORRADE_TARGET_X86 + /* The base tag stays the same, but the extra ones get dropped */ + CORRADE_COMPARE(dispatchExtraFallbackExtra(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2+POPCNT+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackExtra(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt|Cpu::Lzcnt)), "SSE4.2+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackExtra(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt)) { return "AVX2+POPCNT+LZCNT"; } +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Lzcnt)) { return "SSE42+LZCNT"; } +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Ssse3)) { return "SSSE3"; } +const char* dispatchExtraFallbackBase(CORRADE_CPU_DECLARE(Cpu::Popcnt|Cpu::Lzcnt)) { return "POPCNT+LZCNT"; } +#endif + +void CpuTest::tagDispatchExtraFallbackBase() { + #ifdef CORRADE_TARGET_X86 + /* The extra tags stay the same, but the base one gets lowered */ + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Avx512f|Cpu::Popcnt|Cpu::Lzcnt)), "AVX2+POPCNT+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::Lzcnt)), "SSE42+LZCNT"_s); + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Sse42)), "SSSE3"_s); + CORRADE_COMPARE(dispatchExtraFallbackBase(CORRADE_CPU_SELECT(Cpu::Sse3|Cpu::Popcnt|Cpu::Lzcnt)), "POPCNT+LZCNT"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::AvxFma)) { return "AVX2+FMA"; } +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::Avx|Cpu::AvxF16c)) { return "AVX+F16C"; } +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::Avx)) { return "AVX"; } +const char* dispatchExtraFallbackBoth(CORRADE_CPU_DECLARE(Cpu::AvxF16c|Cpu::AvxFma)) { return "F16C+FMA"; } +#endif + +void CpuTest::tagDispatchExtraFallbackBoth() { + #ifdef CORRADE_TARGET_X86 + /* Top-class HW, just pick AVX2 as it's the closest */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Avx512f|Cpu::AvxFma|Cpu::AvxF16c)), "AVX2+FMA"_s); + + /* We have one extra less than required for the AVX2 variant, fall back to + AVX with F16C */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::AvxF16c)), "AVX+F16C"_s); + + /* We have AVX2, but neither of the extra bits, just some irrelevant ones, + take plain AVX */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Avx2|Cpu::Lzcnt|Cpu::Popcnt)), "AVX"_s); + + /* We have only SSE3 but both extra bits, fall back to the scalar version + that has them. Yes, it's silly, but the scalar fallback needs to be + verified. */ + CORRADE_COMPARE(dispatchExtraFallbackBoth(CORRADE_CPU_SELECT(Cpu::Sse3|Cpu::AvxF16c|Cpu::AvxFma)), "F16C+FMA"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +const char* dispatchExtraPriority(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Popcnt|Cpu::Lzcnt)) { + return "SSE4.2+POPCNT+LZCNT"; +} +const char* dispatchExtraPriority(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Popcnt)) { + return "SSE4.2+POPCNT"; +} +const char* dispatchExtraPriority(CORRADE_CPU_DECLARE(Cpu::Sse42|Cpu::Lzcnt)) { + return "SSE4.2+LZCNT"; +} +#endif + +void CpuTest::tagDispatchExtraPriority() { + #ifdef CORRADE_TARGET_X86 + /* The candidate which has the most tags gets picked. OTOH, if it wouldn't + be there, this call would be ambiguous. */ + CORRADE_COMPARE(dispatchExtraPriority(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt|Cpu::Lzcnt)), "SSE4.2+POPCNT+LZCNT"_s); + + /* Both single-extra-tag candidates have the same calculated priority, the + one for which we actually have the feature gets picked */ + CORRADE_COMPARE(dispatchExtraPriority(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Popcnt)), "SSE4.2+POPCNT"_s); + CORRADE_COMPARE(dispatchExtraPriority(CORRADE_CPU_SELECT(Cpu::Sse42|Cpu::Lzcnt)), "SSE4.2+LZCNT"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + /* Not using an argument here since we *don't* want the overload delegating in this case -- it would hide errors when a certain instruction set doesn't have a corresponding overload, as it'd fall back to a parent one. I'm also @@ -613,6 +906,31 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE42 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_POPCNT int callInstructionFor() { + /* Just pocnt alone; using volatile to prevent this from being folded into + a constant */ + volatile unsigned int a = 0x0005c1a6; + unsigned int count = _mm_popcnt_u32(a); + CORRADE_COMPARE(count, 9); + return count; +} +#endif +#ifdef CORRADE_ENABLE_LZCNT +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_LZCNT int callInstructionFor() { + /* Just lzcnt alone; using volatile to prevent this from being folded into + a constant */ + volatile int a = 0x0005c1a6; + unsigned int count = _lzcnt_u32(a); + + /* Also verify that it does the right thing for 0. If misdetected and the + BSR fallback gets used, this would return something random here. */ + CORRADE_COMPARE(_lzcnt_u32(0), 32); + + CORRADE_COMPARE(count, 13); + return count; +} +#endif #ifdef CORRADE_ENABLE_AVX template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX int callInstructionFor() { __m256d a = _mm256_set_pd(5.47, 2.23, 7.62, 0.5); @@ -632,6 +950,46 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX_F16C int callInstructionFor() { + /* Values from Magnum::Math::Test::HalfTest::pack() */ + __m128 a = _mm_set_ps(0.0f, 123.75f, -0.000351512f, 3.0f); + + /* F16C */ + union { + __m128i v; + std::uint16_t s[8]; + } b; + b.v = _mm_cvtps_ph(a, 0); + + CORRADE_COMPARE(b.s[3], 0x0000); + CORRADE_COMPARE(b.s[2], 0x57bc); + CORRADE_COMPARE(b.s[1], 0x8dc2); + CORRADE_COMPARE(b.s[0], 0x4200); + return b.s[0]; +} +#endif +#ifdef CORRADE_ENABLE_AVX_FMA +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX_FMA int callInstructionFor() { + /* Values from Magnum::Math::Test::FunctionsTest::fma() */ + __m128 a = _mm_set_ps(0.0f, 2.0f, 1.5f, 0.5f); + __m128 b = _mm_set_ps(0.0f, 3.0f, 2.0f, -1.0f); + __m128 c = _mm_set_ps(0.0f, 0.75f, 0.25f, 0.1f); + + /* FMA */ + union { + __m128 v; + float s[4]; + } d; + d.v = _mm_fmadd_ps(a, b, c); + + CORRADE_COMPARE(d.s[3], 0.0f); + CORRADE_COMPARE(d.s[2], 6.75f); + CORRADE_COMPARE(d.s[1], 3.25f); + CORRADE_COMPARE(d.s[0], -0.4f); + return d.s[2]; +} +#endif #ifdef CORRADE_ENABLE_AVX2 template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX2 int callInstructionFor() { __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); diff --git a/src/Corrade/Utility/IntrinsicsAvx.h b/src/Corrade/Utility/IntrinsicsAvx.h index 0f9203046..280009cec 100644 --- a/src/Corrade/Utility/IntrinsicsAvx.h +++ b/src/Corrade/Utility/IntrinsicsAvx.h @@ -25,14 +25,15 @@ */ /** @file -@brief Intrinsics for x86 AVX and AVX2 instructions +@brief Intrinsics for x86 LZCNT, AVX, AVX F16C, AVX FMA and AVX2 instructions @m_since_latest Equivalent to @cpp #include @ce on most compilers except for GCC 4.8, where it contains an additional workaround to make the instructions -available with just the @ref CORRADE_ENABLE_AVX or @ref CORRADE_ENABLE_AVX2 -function attributes instead of having to specify `-mavx` or `-mavx2` for the -whole compilation unit. +available with just the @ref CORRADE_ENABLE_AVX, @ref CORRADE_ENABLE_AVX_F16C, +@ref CORRADE_ENABLE_AVX_FMA or @ref CORRADE_ENABLE_AVX2 function attributes +instead of having to specify `-mlzcnt`, `-mavx`, `-mf16c`, `-mfma` or `-mavx2` +for the whole compilation unit. As AVX-512 is supported only since GCC 4.9, which doesn't need this workaround, it's not handled here. @@ -57,6 +58,8 @@ it's not handled here. instructions are included below by defining target("avx2") and directly pulling in , and only doing that on GCC 4.8, as all other compilers have everything already included from the top-level . + Same then goes for F16C, FMA and LZCNT, which all also have such weird + interactions when pulled in together. I wonder what impact this has on optimization, but I don't care that much as GCC 4.8 is mainly for backwards compatibility testing now, and not serious @@ -87,3 +90,45 @@ it's not handled here. #pragma pop_macro("__AVX2__") #pragma GCC pop_options #endif + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("f16c") +#pragma push_macro("__F16C__") +#ifndef __F16C__ +#define __F16C__ +#endif +#include +#pragma pop_macro("__F16C__") +#pragma GCC pop_options +#endif + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("fma") +#pragma push_macro("__FMA__") +#ifndef __FMA__ +#define __FMA__ +#endif +#include +#pragma pop_macro("__FMA__") +#pragma GCC pop_options +#endif + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("lzcnt") +#pragma GCC target("abm") /* GCC 4.8 needs this, 4.9 not; urecognized on Clang */ +#pragma push_macro("__LZCNT__") +#pragma push_macro("__ABM__") +#ifndef __ABM__ +#define __ABM__ +#endif +#ifndef __LZCNT__ +#define __LZCNT__ +#endif +#include +#pragma pop_macro("__ABM__") +#pragma pop_macro("__LZCNT__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/Utility/IntrinsicsSse4.h b/src/Corrade/Utility/IntrinsicsSse4.h index 92d4b9bc7..e79bda952 100644 --- a/src/Corrade/Utility/IntrinsicsSse4.h +++ b/src/Corrade/Utility/IntrinsicsSse4.h @@ -25,14 +25,14 @@ */ /** @file -@brief Intrinsics for x86 SSE4.1 and SSE4.2 instructions +@brief Intrinsics for x86 SSE4.1, SSE4.2 and POPCNT instructions @m_since_latest Equivalent to @cpp #include @ce and @cpp #include @ce on most compilers except for GCC 4.8, where it contains an additional workaround to make the instructions available with just the @ref CORRADE_ENABLE_SSE41 and @ref CORRADE_ENABLE_SSE42 function attributes -instead of having to specify `-msse4.1` and `-msse4.2` for the whole +instead of having to specify `-msse4.1`, `-msse4.2` or `-mpopcnt` for the whole compilation unit. Because GCC puts both the SSE4.1 and the SSE4.2 instructions into the same @@ -76,3 +76,18 @@ together, unlike with other SSE variants. #pragma pop_macro("__SSE4_2__") #pragma GCC pop_options #endif + +/* Unlike with SSE4.1 / SSE4.2 there's no weird interaction with the popcnt + intrinsics, so it could be even in the above block. But for extra safety and + consistency with AVX it's separate. */ +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("popcnt") +#pragma push_macro("__POPCNT__") +#ifndef __POPCNT__ +#define __POPCNT__ +#endif +#include +#pragma pop_macro("__POPCNT__") +#pragma GCC pop_options +#endif From 449ae68589d1f783f9d6514a5b34dec20604499d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 14 Jul 2022 01:00:43 +0200 Subject: [PATCH 14/79] Utility: add a workaround for popcnt also for Clang < 7. Hm, these nasty intrinsics-wrapping headers seem to actually be useful for more than just the ancient GCC 4.8. --- src/Corrade/Cpu.h | 2 +- src/Corrade/Utility/IntrinsicsSse4.h | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 28d9d75c1..208735e5e 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1754,7 +1754,7 @@ may need to specify it together with others. See @m_class{m-note m-info} @par - If you target GCC 4.8, you may also want to use + If you target GCC 4.8 or Clang < 7, you may also want to use @ref Corrade/Utility/IntrinsicsSse4.h instead of @cpp #include @ce to be able to access the intrinsics on this compiler. diff --git a/src/Corrade/Utility/IntrinsicsSse4.h b/src/Corrade/Utility/IntrinsicsSse4.h index e79bda952..a35a93507 100644 --- a/src/Corrade/Utility/IntrinsicsSse4.h +++ b/src/Corrade/Utility/IntrinsicsSse4.h @@ -29,11 +29,14 @@ @m_since_latest Equivalent to @cpp #include @ce and -@cpp #include @ce on most compilers except for GCC 4.8, where it -contains an additional workaround to make the instructions available with just -the @ref CORRADE_ENABLE_SSE41 and @ref CORRADE_ENABLE_SSE42 function attributes -instead of having to specify `-msse4.1`, `-msse4.2` or `-mpopcnt` for the whole -compilation unit. +@cpp #include @ce on most compilers except for: + +- GCC 4.8, where it contains an additional workaround to make the + instructions available with just the @ref CORRADE_ENABLE_SSE41 and + @ref CORRADE_ENABLE_SSE42 function attributes instead of having to specify + `-msse4.1`, `-msse4.2` or `-mpopcnt` for the whole compilation unit. +- Clang < 7, where `__POPCNT__` has to be explicitly defined in order to + access the POPCNT instruction Because GCC puts both the SSE4.1 and the SSE4.2 instructions into the same header and just guards each with a different macro, they have to be included @@ -69,6 +72,14 @@ together, unlike with other SSE variants. #define __SSE4_2__ #endif #endif +/* https://github.com/llvm/llvm-project/commit/092d42557b6c70d32b0b9362e4a8db9566369ddc + says this was an accidental omission, so this should do no harm */ +#if defined(CORRADE_TARGET_CLANG) && __clang_major__ < 7 +#pragma push_macro("__POPCNT__") +#ifndef __POPCNT__ +#define __POPCNT__ +#endif +#endif #include #include #if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 @@ -76,6 +87,9 @@ together, unlike with other SSE variants. #pragma pop_macro("__SSE4_2__") #pragma GCC pop_options #endif +#if defined(CORRADE_TARGET_CLANG) && __clang_major__ < 7 +#pragma pop_macro("__POPCNT__") +#endif /* Unlike with SSE4.1 / SSE4.2 there's no weird interaction with the popcnt intrinsics, so it could be even in the above block. But for extra safety and From c4027cfe4e41c5e67d16667d1809950955c0e540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 14 Jul 2022 12:24:36 +0200 Subject: [PATCH 15/79] Cpu: workaround MSVC 2015 and 2017 template parser warts. This was the easier part of the problem, worse is that the Tags enable_if constructor doesn't compile at all. --- src/Corrade/Cpu.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 208735e5e..64b89fa9e 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -937,11 +937,14 @@ template struct BitCount { /* Calculates an absolute priority index for given tag, which is either a base-2 log "plus one" of its index if it's a base tag, or is 1 if it's an - extra tag */ -template Priority::Index & ExtraTagMask ? 1 : BitIndex::Index & BaseTagMask>::Value*(ExtraTagCount + 1)> constexpr priority(T) { + extra tag. + + MSVC 2015 and 2017 need the extra () inside Priority<>, otherwise they + demand that a typename is used for TypeTraits::Index. Heh. */ +template Priority<(TypeTraits::Index & ExtraTagMask ? 1 : BitIndex::Index & BaseTagMask>::Value*(ExtraTagCount + 1))> constexpr priority(T) { return {}; } -template Priority::Value*(ExtraTagCount + 1) + BitCount<((value & ExtraTagMask) >> ExtraTagBitOffset)>::Value> constexpr priority(Tags) { +template Priority<(BitIndex::Value*(ExtraTagCount + 1) + BitCount<((value & ExtraTagMask) >> ExtraTagBitOffset)>::Value)> constexpr priority(Tags) { static_assert(!((value & BaseTagMask) & ((value & BaseTagMask) - 1)), "more than one base tag used"); /* GCC 4.8 loudly complains about enum comparison if I don't cast, sigh */ static_assert(((value & ExtraTagMask) >> ExtraTagBitOffset) < (1 << static_cast(ExtraTagCount)), "extra tag out of expected bounds"); From 5b83438df25828d8d7665c28594e12eed10d4035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 16 Jul 2022 19:17:50 +0200 Subject: [PATCH 16/79] Cpu: make the template expression build on MSVC without /permissive-. Ah well, one more needless type. Fortunately the fix was this easy, expected a lot more cursing. --- src/Corrade/Cpu.h | 51 +++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 64b89fa9e..97c425fbe 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -835,17 +835,14 @@ enum: unsigned int { #endif }; -/* Holds a compile-time combination of tags. Kept private, since it isn't - really directly needed in user code and it would only lead to confusion. */ -template struct Tags { - enum: unsigned int { Value = value }; - - /* Empty initialization. Should not be needed by public code. */ - constexpr explicit Tags(InitT) {} - - /* Conversion from other tag combination, allowed only if the other is - not a subset */ - template constexpr Tags(Tags, typename std::enable_if< +/* On sane compilers (and on MSVC with /permissive-), these two could be + directly in the Tag constructor enable_if expressions. But MSVC chokes hard + on those ("C2988: unrecognizable template declaration/definition", haha), so + they have to be extracted outside. I could also #ifdef around the workaround + and extract it only for MSVC, but I don't think the code duplication would + be worth it. */ +template struct IsTagConversionAllowed { + enum: bool { Value = /* There should be at most one base tag set in both, if there's none then it's Cpu::Scalar */ !((value & BaseTagMask) & ((value & BaseTagMask) - 1)) && @@ -855,22 +852,38 @@ template struct Tags { (otherValue & BaseTagMask) >= (value & BaseTagMask) && /* The other extra bits should be a superset of this */ ((otherValue & value) & ExtraTagMask) == (value & ExtraTagMask) - >::type* = {}) {} - - /* Conversion from a single tag, allowed only if we're a single bit and - the other is not a subset */ - template constexpr Tags(T, typename std::enable_if< + }; +}; +template struct IsSingleTagConversionAllowed { + enum: bool { Value = /* There should be at most one base tag set in this one, the other satisfies that implicitly as it's constrained by TypeTraits */ !((value & BaseTagMask) & ((value & BaseTagMask) - 1)) && /* The other base tag should be the same or derived (i.e, having same or larger value) */ - (TypeTraits::Index & BaseTagMask) >= (value & BaseTagMask) && + (otherIndex & BaseTagMask) >= (value & BaseTagMask) && /* The other extra bits should be a superset of this. Since a single tag can be only one bit, this condition gets satisfied only either if we're Cpu::Scalar or if we have no extra bits. */ - ((TypeTraits::Index & value) & ExtraTagMask) == (value & ExtraTagMask) - >::type* = {}) {} + ((otherIndex & value) & ExtraTagMask) == (value & ExtraTagMask) + }; +}; + +/* Holds a compile-time combination of tags. Kept private, since it isn't + really directly needed in user code and it would only lead to confusion. */ +template struct Tags { + enum: unsigned int { Value = value }; + + /* Empty initialization. Should not be needed by public code. */ + constexpr explicit Tags(InitT) {} + + /* Conversion from other tag combination, allowed only if the other is + not a subset */ + template constexpr Tags(Tags, typename std::enable_if::Value>::type* = {}) {} + + /* Conversion from a single tag, allowed only if we're a single bit and + the other is not a subset */ + template constexpr Tags(T, typename std::enable_if::Index>::Value>::type* = {}) {} /* A subset of operators on Features, excluding the assignment ones -- since they modify the type, they make no sense here */ From 9a787dbbf2f7a0c91e2dab5b2408a7a52c057800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 14 Jul 2022 14:42:45 +0200 Subject: [PATCH 17/79] Add a note about Apple AMX to CORRADE_TARGET_ARM. --- src/Corrade/Corrade.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index a9ed19bb4..9bd3becec 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -720,6 +720,8 @@ sets. Detection for these will be added when the platforms become more - SVE2, which is a next-generation vector instruction set designed to be a successor to both NEON and SVE, scheduled to appear in production in around 2022 +- AMX, which is Apple's proprietary and patented instruction set available + only on their own hardware @see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Neon}, @ref CORRADE_ENABLE_NEON From 8cfaf2df64b31e8372be83db043c2735bcf1c190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 14 Jul 2022 14:39:28 +0200 Subject: [PATCH 18/79] Cpu: detect ARM features on macOS and iOS. Sadly it seems like Apple won't go with SVE/SVE2 for M3 and newer but rather would further push its proprietary/patented AMX extensions. We can only hope the intrinsics and compiler support get eventually opened, but so far it seems the same as with every other open standard like GL, CL or AL. Let them slowly rot to the point of uselessness first, then swiftly deprecate and replace with a proprietary solution. --- src/Corrade/Cpu.cpp | 58 ++++++++++++++++++++++++++++++++++++++++++++- src/Corrade/Cpu.h | 6 ++--- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index b58dde0d2..ea45aa236 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -58,6 +58,10 @@ #elif defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) #include #include + +/* ARM on macOS / iOS */ +#elif defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) +#include #endif namespace Corrade { namespace Cpu { @@ -129,7 +133,22 @@ Features runtimeFeatures(const unsigned long caps) { } #endif -#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) +/* Helper for getting macOS / iOS ARM properties. Yep, it's stringly typed. */ +#if defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) +namespace { + +int appleSysctlByName(const char* name) { + int value; + std::size_t size = sizeof(value); + /* First pointer/size pair is for querying the value, second is for setting + the value. Returns 0 on success. */ + return sysctlbyname(name, &value, &size, nullptr, 0) ? 0 : value; +} + +} +#endif + +#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && ((defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(CORRADE_TARGET_APPLE))) #if defined(CORRADE_TARGET_X86) && defined(CORRADE_TARGET_GCC) /* GCC 10 complains otherwise, however 4.8 doesn't. Doing it via a function attribute instead of passing -mxsave as detecting target architecture in @@ -249,6 +268,43 @@ Features runtimeFeatures() { /proc/cpuinfo? */ return Implementation::runtimeFeatures(getauxval(AT_HWCAP)); + /* Use sysctlbyname() on ARM on macOS / iOS */ + #elif defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) + unsigned int out = 0; + /* https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics, + especially "funny" is how most of the values are getting rid of the NEON + naming, probably because they want to push their proprietary AMX. + Sigh. */ + #ifdef CORRADE_TARGET_32BIT + /* Apple says I should use hw.optional.AdvSIMD instead tho */ + if(appleSysctlByName("hw.optional.neon")) out |= TypeTraits::Index; + /* On 32bit I have no idea how to query FMA / vfpv4 support, so that'll + only be implied if FP16 is available as well. Since I don't think there are many 32bit iOS devices left, that's not worth bothering with. */ + #else + /* To avoid string operations, on 64bit I just assume NEON and FMA being + present, like in the Linux case. Again, for extra security make use of + the CORRADE_TARGET_ defines (which should be always there on ARM64) */ + out |= + #ifdef CORRADE_TARGET_NEON + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FMA + TypeTraits::Index| + #endif + 0; + #endif + /* Apple says I should use hw.optional.arm.FEAT_FP16 instead tho */ + if(appleSysctlByName("hw.optional.neon_fp16")) { + /* As noted above, if FP16 is available on 32bit, bite the bullet and + assume FMA is there as well */ + #ifdef CORRADE_TARGET_32BIT + out |= TypeTraits::Index; + #endif + out |= TypeTraits::Index; + } + + return Features{out}; + /* No other implementation at the moment. The function should not be even defined here in that case -- it's inlined in the header instead. */ #else diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 97c425fbe..703c20afd 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1251,7 +1251,7 @@ class Features { private: template friend constexpr Features features(); friend constexpr Features compiledFeatures(); - #if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) + #if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE)) /* MSVC demands the export macro to be here as well */ friend CORRADE_UTILITY_EXPORT Features runtimeFeatures(); #endif @@ -1497,7 +1497,7 @@ following flags are checked either. On compilers other than GCC, Clang and MSVC the function is @cpp constexpr @ce and delegates into @ref compiledFeatures(). On @ref CORRADE_TARGET_ARM "ARM" and Linux or Android API level 18+ uses -@m_class{m-doc-external} [getauxval()](https://man.archlinux.org/man/getauxval.3) +@m_class{m-doc-external} [getauxval()](https://man.archlinux.org/man/getauxval.3), or on ARM macOS and iOS uses @m_class{m-doc-external} [sysctlbyname()](https://developer.apple.com/documentation/kernel/1387446-sysctlbyname) to check for the @ref Neon, @ref NeonFma and @ref NeonFp16. @ref Neon and @ref NeonFma are implicitly supported on ARM64. On other platforms the function is @cpp constexpr @ce and delegates into @ref compiledFeatures(). @@ -1514,7 +1514,7 @@ value is equal to @ref Scalar, which in turn is equivalent to empty (or default-constructed) @ref Features. @see @ref DefaultBase, @ref DefaultExtra, @ref Default */ -#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(DOXYGEN_GENERATING_OUTPUT) +#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && ((defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(CORRADE_TARGET_APPLE))) || defined(DOXYGEN_GENERATING_OUTPUT) CORRADE_UTILITY_EXPORT Features runtimeFeatures(); #else constexpr Features runtimeFeatures() { return compiledFeatures(); } From 7264d19a92b62f0418f0bb85d7db61fa2f7651aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 15 Jul 2022 12:47:20 +0200 Subject: [PATCH 19/79] Cpu: Doxygen! Bad Doxygen. --- src/Corrade/Cpu.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 703c20afd..bf72cb0cc 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -819,8 +819,12 @@ constexpr Simd128T Simd128{Implementation::Init}; namespace Implementation { +#ifndef DOXYGEN_GENERATING_OUTPUT +/* "warning: Detected potential recursive class relation between..." I DON'T + CARE, DOXYGEN, IT'S NOT YOUR JOB TO PARSE THE CODE IN THIS NAMESPACE */ template struct Priority: Priority {}; template<> struct Priority<0> {}; +#endif /* Count of "extra" tags that are not in the hierarchy. Should not be larger than strictly necessary as it deepens inheritance hierarchy when picking From ebb631a5fa063ba4b01e48a5fcb6b9f32990f69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 16 Jul 2022 19:12:36 +0200 Subject: [PATCH 20/79] Cpu: add bitwise operators also for Tags<> on both sides. Otherwise it decays to an int and ... the horror! --- src/Corrade/Cpu.h | 9 +++++++++ src/Corrade/Test/CpuTest.cpp | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index bf72cb0cc..cac6dc2bc 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -891,12 +891,21 @@ template struct Tags { /* A subset of operators on Features, excluding the assignment ones -- since they modify the type, they make no sense here */ + template constexpr Tags operator|(Tags) const { + return Tags{Init}; + } template constexpr Tags::Index> operator|(U) const { return Tags::Index>{Init}; } + template constexpr Tags operator&(Tags) const { + return Tags{Init}; + } template constexpr Tags::Index> operator&(U) const { return Tags::Index>{Init}; } + template constexpr Tags operator^(Tags) const { + return Tags{Init}; + } template constexpr Tags::Index> operator^(U) const { return Tags::Index>{Init}; } diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index cf99b0287..718280ff4 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -347,6 +347,13 @@ void CpuTest::featuresOperatorOr() { CORRADE_COMPARE(std::uint32_t(cTags2), 7); CORRADE_VERIFY(!std::is_same::value); CORRADE_VERIFY(std::is_same::value); + + /* And also with Tags<> on both sides, check that it doesn't decay to an + int or other horrible thing */ + constexpr auto cTags3 = Cpu::Ssse3|Cpu::Sse41; + constexpr auto cTags4 = cTags3|cTags; + CORRADE_COMPARE(std::uint32_t(cTags4), 15); + CORRADE_VERIFY(std::is_same>::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -387,6 +394,13 @@ void CpuTest::featuresOperatorAnd() { CORRADE_COMPARE(std::uint32_t(cTags2), 8); CORRADE_VERIFY(!std::is_same::value); CORRADE_VERIFY(std::is_same::value); + + /* And also with Tags<> on both sides, check that it doesn't decay to an + int or other horrible thing */ + constexpr auto cTags3 = Cpu::Ssse3|Cpu::Sse41; + constexpr auto cTags4 = cTags3 & cTags; + CORRADE_COMPARE(std::uint32_t(cTags4), 8); + CORRADE_VERIFY(std::is_same>::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif @@ -428,6 +442,13 @@ void CpuTest::featuresOperatorXor() { CORRADE_COMPARE(std::uint32_t(cTags2), 10); CORRADE_VERIFY(!std::is_same::value); CORRADE_VERIFY(std::is_same::value); + + /* And also with Tags<> on both sides, check that it doesn't decay to an + int or other horrible thing */ + constexpr auto cTags3 = Cpu::Ssse3|Cpu::Sse41; + constexpr auto cTags4 = cTags3 ^ cTags; + CORRADE_COMPARE(std::uint32_t(cTags4), 7); + CORRADE_VERIFY(std::is_same>::value); #else CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif From c2bb1673474f1d148d6d2740cdd705884686f841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 16 Jul 2022 19:05:21 +0200 Subject: [PATCH 21/79] Cpu: macros for efficient automatic runtime dispatch. --- doc/snippets/Corrade.cpp | 56 +++++++++ src/Corrade/Cpu.h | 215 ++++++++++++++++++++++++++++++++++ src/Corrade/Test/CpuTest.cpp | 217 +++++++++++++++++++++++++++++++++++ 3 files changed, 488 insertions(+) diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index 48fbd8409..de901f87a 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -79,6 +79,50 @@ int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIP } #endif /* [Cpu-usage-target-attributes] */ + +namespace Bar { +using TransformT = void(*)(Containers::ArrayView); +TransformT transformImplementation(Cpu::ScalarT); +TransformT transformImplementation(Cpu::Sse42T); +TransformT transformImplementation(Cpu::Avx2T); +TransformT transformImplementation(Cpu::Features); +/* [Cpu-usage-automatic-runtime-dispatch-declare] */ +using TransformT = void(*)(Containers::ArrayView); + +TransformT transformImplementation(Cpu::ScalarT) { + return [](Containers::ArrayView data) { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} +TransformT transformImplementation(Cpu::Sse42T) { + return [](Containers::ArrayView data) { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} +TransformT transformImplementation(Cpu::Avx2T) { + return [](Containers::ArrayView data) { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} + +CORRADE_CPU_DISPATCHER_BASE(TransformT, transformImplementation) +/* [Cpu-usage-automatic-runtime-dispatch-declare] */ + +using LookupT = int(*)(int); +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)); +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2)); +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt)); +LookupT lookupImplementation(Cpu::Features); +/* [Cpu-usage-automatic-runtime-dispatch-extra-declare] */ +using LookupT = int(*)(DOXYGEN_ELLIPSIS(int)); + +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + DOXYGEN_ELLIPSIS(return {};) +} +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2)) { + DOXYGEN_ELLIPSIS(return {};) +} +LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt)) { + DOXYGEN_ELLIPSIS(return {};) +} + +CORRADE_CPU_DISPATCHER(LookupT, lookupImplementation, Cpu::Popcnt, Cpu::Lzcnt) +/* [Cpu-usage-automatic-runtime-dispatch-extra-declare] */ +} #endif inline void foo(Cpu::ScalarT) {} @@ -114,6 +158,18 @@ int found = lookup(CORRADE_CPU_SELECT(Cpu::Default), DOXYGEN_ELLIPSIS(0)); static_cast(found); } +{ +using namespace Bar; +Containers::ArrayView data; +/* [Cpu-usage-automatic-runtime-dispatch-call] */ +/* Dispatch once and cache the function pointer */ +TransformT transform = transformImplementation(Cpu::runtimeFeatures()); + +/* Call many times */ +transform(data); +/* [Cpu-usage-automatic-runtime-dispatch-call] */ +} + { /* [Cpu-tag-from-type] */ foo(Cpu::Avx2); diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index cac6dc2bc..247cb29b2 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -31,6 +31,7 @@ * @m_since_latest */ +#include "Corrade/Utility/Macros.h" #include "Corrade/Utility/Utility.h" #include "Corrade/Utility/visibility.h" @@ -152,6 +153,11 @@ returned @ref Cpu::Features against particular tags to decide which variant to @snippet Corrade.cpp Cpu-usage-runtime-manual-dispatch +While such approach gives you the most control, manually managing the dispatch +branches is error prone and the argument passthrough may also add nontrivial +overhead. See below for an +@ref Cpu-usage-automatic-runtime-dispatch "efficient automatic runtime dispatch". + @section Cpu-usage-extra Usage with extra instruction sets Besides the base instruction set, which on x86 is @ref Sse2 through @@ -255,6 +261,44 @@ explicitly enabled as well, in contrast a scalar variant would have no target-specific annotations at all. @snippet Corrade.cpp Cpu-usage-target-attributes + +@section Cpu-usage-automatic-runtime-dispatch Automatic runtime dispatch + +Similarly to how the best-matching function variant can be picked at compile +time, there's a possibility to do the same at runtime without maintaining a +custom dispatch code for each case +@ref Cpu-usage-dispatch-runtime "as was shown above". To avoid having to +dispatch on every call and to remove the argument passthrough overhead, all +variants need to have the same function signature, separate from the CPU tags. +That's achievable by putting them into lambdas with a common signature, and +returning that lambda from a wrapper function that contains the CPU tag. After +that, a runtime dispatcher function that is created with the +@ref CORRADE_CPU_DISPATCHER_BASE() macro. The @cpp transform() @ce variants +from above would then look like this instead: + +@snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-declare + +The macro creates an overload of the same name, but taking @ref Features +instead, and internally dispatches to one of the overloads using the same rules +as in the compile-time dispatch. Which means you can now call it with e.g. +@ref runtimeFeatures(), get a function pointer back and then call it with the +actual arguments: + +@snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-call + +@subsection Cpu-usage-automatic-runtime-dispatch-extra Automatic runtime dispach with extra instruction sets + +If the variants are tagged with extra instruction sets instead of just the +base instruction set like in the @cpp lookup() @ce case +@ref Cpu-usage-extra "shown above", you'll use the @ref CORRADE_CPU_DISPATCHER() +macro instead. There, to avoid a combinatorial explosion of cases to check, +you're expected to list the actual extra tags the overloads use. Which is +usually just one or two out of the whole set: + +@snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-extra-declare + +On the call side, there's no difference. The created dispatcher function takes +@ref Features as well. */ namespace Cpu { @@ -1564,6 +1608,177 @@ best viable overload. */ #define CORRADE_CPU_SELECT(tag) tag, Corrade::Cpu::Implementation::priority(tag) +/** +@brief Create a function for a runtime dispatch on a base CPU instruction set +@m_since_latest + +Given a set of function overloads named @p function that accept a CPU tag as a +parameter and return a function pointer of @p type, creates a function with +signature @cpp type function(Cpu::Features) @ce which will select among the +overloads using a runtime-specified @relativeref{Corrade,Cpu::Features}, using +the same rules as the compile-time overload selection. For this macro to work, +at the very least there has to be an overload with a +@relativeref{Corrade,Cpu::ScalarT} argument. See +@ref Cpu-usage-automatic-runtime-dispatch for more information and an example. + +This function works with just a single base CPU instruction tag such as +@relativeref{Corrade,Cpu::Avx2} or @relativeref{Corrade,Cpu::Neon}, but not the +extra instruction sets like @relativeref{Corrade,Cpu::Lzcnt} or +@relativeref{Corrade,Cpu::AvxFma}. For a dispatch that takes extra instruction +sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. +*/ +/* Ideally this would reuse _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(), + unfortunately due to MSVC not being able to defer macro calls unless + "the new preprocessor" is enabled it wouldn't be possible to get rid of the + CORRADE_CPU_SELECT() macro from there. */ +#ifdef CORRADE_TARGET_X86 +#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ + type function(Corrade::Cpu::Features features) { \ + if(features & Corrade::Cpu::Avx512f) \ + return function(Corrade::Cpu::Avx512f); \ + if(features & Corrade::Cpu::Avx2) \ + return function(Corrade::Cpu::Avx2); \ + if(features & Corrade::Cpu::Avx) \ + return function(Corrade::Cpu::Avx); \ + if(features & Corrade::Cpu::Sse42) \ + return function(Corrade::Cpu::Sse42); \ + if(features & Corrade::Cpu::Sse41) \ + return function(Corrade::Cpu::Sse41); \ + if(features & Corrade::Cpu::Ssse3) \ + return function(Corrade::Cpu::Ssse3); \ + if(features & Corrade::Cpu::Sse3) \ + return function(Corrade::Cpu::Sse3); \ + if(features & Corrade::Cpu::Sse2) \ + return function(Corrade::Cpu::Sse2); \ + return function(Corrade::Cpu::Scalar); \ + } +#elif defined(CORRADE_TARGET_ARM) +#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ + type function(Corrade::Cpu::Features features) { \ + if(features & Corrade::Cpu::NeonFp16) \ + return function(Corrade::Cpu::NeonFp16); \ + if(features & Corrade::Cpu::NeonFma) \ + return function(Corrade::Cpu::NeonFma); \ + if(features & Corrade::Cpu::Neon) \ + return function(Corrade::Cpu::Neon); \ + return function(Corrade::Cpu::Scalar); \ + } +#elif defined(CORRADE_TARGET_WASM) +#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ + type function(Corrade::Cpu::Features features) { \ + if(features & Corrade::Cpu::Simd128) \ + return function(Corrade::Cpu::Simd128); \ + return function(Corrade::Cpu::Scalar); \ + } +#else +#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ + type function(Corrade::Cpu::Features features) { \ + return function(Corrade::Cpu::Scalar); \ + } +#endif + +#ifndef DOXYGEN_GENERATING_OUTPUT +#ifdef CORRADE_TARGET_X86 +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + if(features >= (Corrade::Cpu::Avx512f extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Avx512f extra)); \ + if(features >= (Corrade::Cpu::Avx2 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Avx2 extra)); \ + if(features >= (Corrade::Cpu::Avx extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Avx extra)); \ + if(features >= (Corrade::Cpu::Sse42 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse42 extra)); \ + if(features >= (Corrade::Cpu::Sse41 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse41 extra)); \ + if(features >= (Corrade::Cpu::Ssse3 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Ssse3 extra)); \ + if(features >= (Corrade::Cpu::Sse3 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse3 extra)); \ + if(features >= (Corrade::Cpu::Sse2 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Sse2 extra)); \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#elif defined(CORRADE_TARGET_ARM) +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + if(features >= (Corrade::Cpu::NeonFp16 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::NeonFp16 extra)); \ + if(features >= (Corrade::Cpu::NeonFma extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::NeonFma extra)); \ + if(features >= (Corrade::Cpu::Neon extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Neon extra)); \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#elif defined(CORRADE_TARGET_WASM) +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + if(features >= (Corrade::Cpu::Simd128 extra)) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Simd128 extra)); \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#else +#define _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, extra) \ + return function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar extra)); +#endif + +/* CORRADE_CPU_DISPATCHER() specialization for 0 extra instruction sets. + Basically equivalent to CORRADE_CPU_DISPATCHER_BASE() except for the extra + CORRADE_CPU_SELECT() macro. */ +#define _CORRADE_CPU_DISPATCHER0(type, function) \ + type function(Corrade::Cpu::Features features) { \ + _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, ) \ + } + +/* CORRADE_CPU_DISPATCHER() specialization for 1+ extra instruction sets. On + Clang this still generates quite a reasonable code for 2 extra sets compared + to CORRADE_CPU_DISPATCHER_BASE(), on GCC it's ~25% longer but also still + reasonable. I attempted adding an "unrolled" _CORRADE_CPU_DISPATCHER2() but + it made everything significantly worse on both compilers, more than doubling + the amount of generated code. */ +#define _CORRADE_CPU_DISPATCHERn(type, function, ...) \ + template CORRADE_ALWAYS_INLINE type function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags) { \ + _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, |Corrade::Cpu::Implementation::Tags{Corrade::Cpu::Implementation::Init}) \ + } \ + template CORRADE_ALWAYS_INLINE type function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags extra, First first, Next... next) { \ + static_assert(!(static_cast(Corrade::Cpu::Implementation::tags(First{Corrade::Cpu::Implementation::Init})) & Corrade::Cpu::Implementation::BaseTagMask), \ + "only extra instruction set tags should be explicitly listed"); \ + if(features & first) \ + return function ## Internal(features, extra|first, next...); \ + else \ + return function ## Internal(features, extra, next...); \ + } \ + type function(Corrade::Cpu::Features features) { \ + return function ## Internal(features, Corrade::Cpu::Implementation::Tags<0>{Corrade::Cpu::Implementation::Init}, __VA_ARGS__); \ + } + +/* Called from CORRADE_CPU_DISPATCHER() to pick either _CORRADE_CPU_DISPATCHER0 + if there are no extra arguments after function or _CORRADE_CPU_DISPATCHERn + if there is 1 to 7 extra args (assuming there isn't a need to have more than + 7). Source: https://stackoverflow.com/a/11763277 */ +#define _CORRADE_CPU_DISPATCHER_PICK(_0, _1, _2, _3, _4, _5, _6, _7, macroName, ...) macroName +#endif + +/** +@brief Create a function for a runtime dispatch on a base CPU instruction set and select extra instruction sets +@m_since_latest + +Given a set of function overloads named @p function that accept a CPU tag +combination wrapped in @ref CORRADE_CPU_DECLARE() as a parameter and return a +function pointer of @p type, creates a function with signature +@cpp type function(Cpu::Features) @ce which will select among the overloads +using a runtime-specified @relativeref{Corrade,Cpu::Features}, using the same +rules as the compile-time overload selection. The extra instruction sets +considered in the overload selection are specified as additional parameters to +the macro, specifying none is valid as well. For this macro to work, at the +very least there has to be an overload with a +@ref Corrade::Cpu::Scalar "CORRADE_CPU_DECLARE(Cpu::Scalar)" argument. See +@ref Cpu-usage-automatic-runtime-dispatch for more information and an example. + +For a dispatch using just the base instruction set use +@ref CORRADE_CPU_DISPATCHER_BASE() instead. +*/ +#ifdef DOXYGEN_GENERATING_OUTPUT +#define CORRADE_CPU_DISPATCHER(type, function, ...) +#else +#define CORRADE_CPU_DISPATCHER(type, ...) \ + _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(type, __VA_ARGS__) +#endif + #if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) /** @brief Enable SSE2 for given function diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index 718280ff4..5152a4037 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -96,6 +96,11 @@ struct CpuTest: TestSuite::Tester { void tagDispatchExtraFallbackBoth(); void tagDispatchExtraPriority(); + void tagDispatchRuntime(); + void tagDispatchRuntimeExtra(); + void tagDispatchRuntimeExtraCombination(); + void tagDispatchRuntimeExtraZeroExtra(); + template void enableMacros(); void debug(); @@ -145,6 +150,11 @@ CpuTest::CpuTest() { &CpuTest::tagDispatchExtraFallbackBoth, &CpuTest::tagDispatchExtraPriority, + &CpuTest::tagDispatchRuntime, + &CpuTest::tagDispatchRuntimeExtra, + &CpuTest::tagDispatchRuntimeExtraCombination, + &CpuTest::tagDispatchRuntimeExtraZeroExtra, + #ifdef CORRADE_TARGET_X86 &CpuTest::enableMacros, &CpuTest::enableMacros, @@ -825,6 +835,213 @@ void CpuTest::tagDispatchExtraPriority() { #endif } +/* The lambda wrappers are marked with CORRADE_ALWAYS_INLINE to make them go + away with optimizations */ +#if defined(CORRADE_TARGET_X86) || defined(CORRADE_TARGET_ARM) || defined(CORRADE_TARGET_WASM) +typedef const char*(*DispatchRuntimeT)(); + +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::ScalarT) { + return []() -> const char* { return "scalar"; }; +} +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::Sse3T) { + return []() -> const char* { return "SSE3"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::Avx2T) { + return []() -> const char* { return "AVX2"; }; +} +#elif defined(CORRADE_TARGET_ARM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::NeonT) { + return []() -> const char* { return "NEON"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::NeonFmaT) { + return []() -> const char* { return "NEON FMA"; }; +} +#elif defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::Simd128T) { + return []() -> const char* { return "SIMD128"; }; +} +#else +#error +#endif + +CORRADE_CPU_DISPATCHER_BASE(DispatchRuntimeT, dispatchRuntime) +#endif + +void CpuTest::tagDispatchRuntime() { + /* Explicitly casting to Cpu::Features here to ensure it's indeed going + through the runtime switch and not directly. Otherwise it's the same as + tagDispatch(). */ + + #ifdef CORRADE_TARGET_X86 + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Avx512f})(), "AVX2"_s); + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Sse42})(), "SSE3"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Sse3})(), "SSE3"_s); + + /* Anything below gets ... the scalar */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Sse2})(), "scalar"_s); + #elif defined(CORRADE_TARGET_ARM) + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::NeonFp16})(), "NEON FMA"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Neon})(), "NEON"_s); + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #elif defined(CORRADE_TARGET_WASM) + /* Exact match. No other opportunity to test anything, but better than have + the macro completely untested. */ + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Simd128})(), "SIMD128"_s); + CORRADE_COMPARE(dispatchRuntime(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> const char* { return "scalar"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::AvxF16c|Cpu::AvxFma)) { + return []() -> const char* { return "F16C+FMA"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Avx)) { + return []() -> const char* { return "AVX"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Avx|Cpu::AvxF16c)) { + return []() -> const char* { return "AVX+F16C"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::AvxFma)) { + return []() -> const char* { return "AVX2+FMA"; }; +} + +CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtra, Cpu::AvxF16c, Cpu::AvxFma) +#endif + +void CpuTest::tagDispatchRuntimeExtra() { + /* Explicitly casting to Cpu::Features here to ensure it's indeed going + through the runtime switch and not directly. Otherwise it's mostly the + same as tagDispatchExtraFallbackBoth(). */ + + #ifdef CORRADE_TARGET_X86 + /* Top-class HW, just pick AVX2 as it's the closest */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Avx512f|Cpu::AvxFma|Cpu::AvxF16c})(), "AVX2+FMA"_s); + + /* We have one extra less than required for the AVX2 variant, fall back to + AVX with F16C */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Avx2|Cpu::AvxF16c})(), "AVX+F16C"_s); + + /* We have AVX2, but neither of the extra bits, just some irrelevant ones, + take plain AVX */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Avx2|Cpu::Lzcnt|Cpu::Popcnt})(), "AVX"_s); + + /* We have only SSE3 but both extra bits, fall back to the scalar version + that has them. Yes, it's silly, but the scalar fallback needs to be + verified. */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Sse3|Cpu::AvxF16c|Cpu::AvxFma})(), "F16C+FMA"_s); + + /* Finally, this will fall back to the scalar version. */ + CORRADE_COMPARE(dispatchRuntimeExtra(Cpu::Features{Cpu::Sse3})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraCombination(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> const char* { return "scalar"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraCombination(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt)) { + return []() -> const char* { return "SSE4.1+POPCNT+LZCNT"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraCombination(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Popcnt|Cpu::Lzcnt)) { + return []() -> const char* { return "AVX2+POPCNT+LZCNT"; }; +} + +CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraCombination, Cpu::Popcnt|Cpu::Lzcnt) +#endif + +void CpuTest::tagDispatchRuntimeExtraCombination() { + /* This verifies that the CORRADE_CPU_DISPATCHER() can accept also + tag combinations, in case the variants only ever use them together */ + + #ifdef CORRADE_TARGET_X86 + /* Verify the tag combinations don't get skipped if we have them all or + more */ + CORRADE_COMPARE(dispatchRuntimeExtraCombination(Cpu::Features{Cpu::Avx512f|Cpu::Popcnt|Cpu::Lzcnt|Cpu::AvxFma})(), "AVX2+POPCNT+LZCNT"_s); + CORRADE_COMPARE(dispatchRuntimeExtraCombination(Cpu::Features{Cpu::Avx|Cpu::Popcnt|Cpu::Lzcnt})(), "SSE4.1+POPCNT+LZCNT"_s); + + /* But also that they don't get picked if we don't have them all */ + CORRADE_COMPARE(dispatchRuntimeExtraCombination(Cpu::Features{Cpu::Avx512f|Cpu::Popcnt|Cpu::AvxFma})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + +#if defined(CORRADE_TARGET_X86) || defined(CORRADE_TARGET_ARM) || defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> const char* { return "scalar"; }; +} + +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Sse2)) { + return []() -> const char* { return "SSE2"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Avx)) { + return []() -> const char* { return "AVX"; }; +} +CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraZeroExtra) +#elif defined(CORRADE_TARGET_ARM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Neon)) { + return []() -> const char* { return "NEON"; }; +} +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::NeonFma)) { + return []() -> const char* { return "NEON FMA"; }; +} + +CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraZeroExtra) +#elif defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Simd128)) { + return []() -> const char* { return "SIMD128"; }; +} + +CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraZeroExtra) +#else +#error +#endif +#endif + +void CpuTest::tagDispatchRuntimeExtraZeroExtra() { + /* Explicitly casting to Cpu::Features here to ensure it's indeed going + through the runtime switch and not directly. Otherwise it's mostly the + same as tagDispatchRuntime(). */ + + #ifdef CORRADE_TARGET_X86 + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Avx512f})(), "AVX"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Sse2})(), "SSE2"_s); + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #elif defined(CORRADE_TARGET_ARM) + /* If no match, gets the next highest available */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::NeonFp16})(), "NEON FMA"_s); + + /* Exact match */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Neon})(), "NEON"_s); + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #elif defined(CORRADE_TARGET_WASM) + /* Exact match. No other opportunity to test anything, but better than have + the macro completely untested. */ + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Simd128})(), "SIMD128"_s); + CORRADE_COMPARE(dispatchRuntimeExtraZeroExtra(Cpu::Features{Cpu::Scalar})(), "scalar"_s); + #else + CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); + #endif +} + /* Not using an argument here since we *don't* want the overload delegating in this case -- it would hide errors when a certain instruction set doesn't have a corresponding overload, as it'd fall back to a parent one. I'm also From bccd27bddcf635d9788e68319226279f9850f2ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 17 Jul 2022 16:02:46 +0200 Subject: [PATCH 22/79] Cpu: another day, another case of MSVC NOT WORKING AT ALL. --- src/Corrade/Cpu.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 247cb29b2..b78441b59 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1774,9 +1774,16 @@ For a dispatch using just the base instruction set use */ #ifdef DOXYGEN_GENERATING_OUTPUT #define CORRADE_CPU_DISPATCHER(type, function, ...) -#else +#elif !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_CPU_DISPATCHER(type, ...) \ _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(type, __VA_ARGS__) +#else +/* Workaround for MSVC not being able to expand __VA_ARGS__ correctly. Would + work with /Zc:preprocessor or /experimental:preprocessor, but I'm not + enabling that globally yet. Source: https://stackoverflow.com/a/5134656 */ +#define _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS(x) x +#define CORRADE_CPU_DISPATCHER(type, ...) \ + _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS( _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(type, __VA_ARGS__)) #endif #if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) From ac9d940a37e30d9c71eefb2a666db8505c5b7484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 17 Jul 2022 17:33:30 +0200 Subject: [PATCH 23/79] doc: update outdated reference to CORRADE_TARGET_* variables. --- doc/corrade-cmake.dox | 7 +++++-- modules/FindCorrade.cmake | 9 +++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/corrade-cmake.dox b/doc/corrade-cmake.dox index ddf3f9c7b..63aae334f 100644 --- a/doc/corrade-cmake.dox +++ b/doc/corrade-cmake.dox @@ -239,8 +239,11 @@ also available as preprocessor variables if you include used for colored output with @ref Utility::Debug on Windows Besides all the defines above, the @ref Corrade/Corrade.h additionally defines -@ref CORRADE_CXX_STANDARD, @ref CORRADE_TARGET_X86, @ref CORRADE_TARGET_ARM, -@ref CORRADE_TARGET_POWERPC, @ref CORRADE_TARGET_LIBCXX, +@ref CORRADE_CXX_STANDARD, @ref CORRADE_TARGET_X86 and related x86-specific +variables, @ref CORRADE_TARGET_ARM and related ARM-specific variables, +@ref CORRADE_TARGET_POWERPC, @ref CORRADE_TARGET_WASM and related +WebAssembly-specific variables, @ref CORRADE_TARGET_32BIT, +@ref CORRADE_TARGET_BIG_ENDIAN, @ref CORRADE_TARGET_LIBCXX, @ref CORRADE_TARGET_LIBSTDCXX and @ref CORRADE_TARGET_DINKUMWARE based on target architecture, standard and STL used. They are not exposed in CMake because the meaning is unclear with projects that mix more different C++ diff --git a/modules/FindCorrade.cmake b/modules/FindCorrade.cmake index ff3429b2b..714506249 100644 --- a/modules/FindCorrade.cmake +++ b/modules/FindCorrade.cmake @@ -327,10 +327,11 @@ set(_corradeFlags TARGET_WINDOWS_RT TARGET_EMSCRIPTEN TARGET_ANDROID - # TARGET_X86 etc and TARGET_LIBCXX are not exposed to CMake as the meaning - # is unclear on platforms with multi-arch binaries or when mixing different - # STL implementations. TARGET_GCC etc are figured out via UseCorrade.cmake, - # as the compiler can be different when compiling the lib & when using it. + # TARGET_X86 etc, TARGET_32BIT, TARGET_BIG_ENDIAN and TARGET_LIBCXX etc. + # are not exposed to CMake as the meaning is unclear on platforms with + # multi-arch binaries or when mixing different STL implementations. + # TARGET_GCC etc are figured out via UseCorrade.cmake, as the compiler can + # be different when compiling the lib & when using it. PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT TESTSUITE_TARGET_XCTEST UTILITY_USE_ANSI_COLORS) From 71398f32811fef634a4c15ce1ee3a22c67c33638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 17 Jul 2022 20:25:44 +0200 Subject: [PATCH 24/79] Cpu: automatic cached dispatch via a pointer and via GNU IFUNC. Probably something I should have expected, but *wow* IFUNC needs so much more work than just slapping an attribute on a function. The called-early restrictions are very strict. --- CMakeLists.txt | 24 ++++++ doc/building-corrade.dox | 7 ++ doc/corrade-cmake.dox | 3 + doc/snippets/Corrade.cpp | 37 +++++++++ modules/FindCorrade.cmake | 3 + src/Corrade/Corrade.h | 14 ++++ src/Corrade/Cpu.h | 146 +++++++++++++++++++++++++++++++++ src/Corrade/Test/CpuTest.cpp | 148 ++++++++++++++++++++++++++++++++++ src/Corrade/configure.h.cmake | 1 + 9 files changed, 383 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 63e60eb61..f353b0748 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,6 +187,30 @@ if(CORRADE_TARGET_WINDOWS) endif() endif() +# Check if we can use IFUNC for CPU dispatch. Linux with glibc and Android with +# API 18+ has it, but e.g. Alpine Linux with musl doesn't, and on Android with +# API < 30 we don't get AT_HWCAP passed into the resolver and can't call +# getauxval() ourselves because it's too early at that point, which makes it +# pretty useless. Plus it also needs a certain binutils version and a capable +# compiler, so it's easiest to just verify the whole thing. +if(CORRADE_TARGET_UNIX) + include(CheckCXXSourceCompiles) + check_cxx_source_compiles("\ +int fooImplementation() { return 42; } +#if defined(__ANDROID_API__) && __ANDROID_API__ < 30 +#error need Android API 30+ to have AT_HWCAP passed into the resolver +#endif +extern \"C\" int(*fooDispatcher())() { + return fooImplementation; +} +int foo() __attribute__((ifunc(\"fooDispatcher\"))); +int main() { return foo() - 42; }\ + " _CORRADE_CPU_CAN_USE_IFUNC) +else() + set(_CORRADE_CPU_CAN_USE_IFUNC OFF) +endif() +cmake_dependent_option(CORRADE_CPU_USE_IFUNC "Allow using GNU IFUNC for runtime CPU dispatch" ON "_CORRADE_CPU_CAN_USE_IFUNC" OFF) + # Backwards compatibility for unprefixed CMake options. If the user isn't # explicitly using prefixed options in the first run already, accept the # unprefixed options, and remember this decision for subsequent runs diff --git a/doc/building-corrade.dox b/doc/building-corrade.dox index 94b8cb5a3..32fb32d9c 100644 --- a/doc/building-corrade.dox +++ b/doc/building-corrade.dox @@ -553,6 +553,13 @@ Options controlling the build: Platform-specific options: +- `CORRADE_CPU_USE_IFUNC` --- Allow + [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) to be used for + runtime dispatch in the @ref Cpu library. Available only on platforms that + support it, which is currently only Linux with glibc and Android with API + 30+, and there enabled by default. See + @ref Cpu-usage-automatic-cached-dispatch for details and information about + performance tradeoffs. - `CORRADE_UTILITY_USE_ANSI_COLORS` --- if building for Windows, this will use ANSI escape codes in @ref Utility::Debug instead of WINAPI functions. Note that you need at least Windows 10 or non-standard console emulator to diff --git a/doc/corrade-cmake.dox b/doc/corrade-cmake.dox index 63aae334f..27ed7da6d 100644 --- a/doc/corrade-cmake.dox +++ b/doc/corrade-cmake.dox @@ -230,6 +230,9 @@ also available as preprocessor variables if you include - `CORRADE_TARGET_MSVC` --- Defined if compiling with MSVC or Clang with a MSVC frontend - `CORRADE_TARGET_MINGW` --- Defined if compiling under MinGW +- `CORRADE_CPU_USE_IFUNC` - Defined if + [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) is allowed to be + used for runtime dispatch in the @ref Cpu library - `CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT` --- Defined if @ref PluginManager doesn't support dynamic plugin loading due to platform limitations diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index de901f87a..3b03740a9 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -122,6 +122,26 @@ LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzc CORRADE_CPU_DISPATCHER(LookupT, lookupImplementation, Cpu::Popcnt, Cpu::Lzcnt) /* [Cpu-usage-automatic-runtime-dispatch-extra-declare] */ + +#ifdef CORRADE_CPU_USE_IFUNC +/* [Cpu-usage-automatic-cached-dispatch-ifunc] */ +CORRADE_CPU_DISPATCHED_IFUNC(LookupT, lookupImplementation, int lookup(DOXYGEN_ELLIPSIS(int))) +/* [Cpu-usage-automatic-cached-dispatch-ifunc] */ +#else +/* [Cpu-usage-automatic-cached-dispatch-pointer] */ +CORRADE_CPU_DISPATCHED_POINTER(LookupT, lookupImplementation, lookup) +/* [Cpu-usage-automatic-cached-dispatch-pointer] */ +#endif + +namespace BarInsideABar { +int lookup(DOXYGEN_ELLIPSIS(int)); +/* [Cpu-usage-automatic-cached-dispatch-compile-time] */ +int lookup(DOXYGEN_ELLIPSIS(int)) { + return lookupImplementation(CORRADE_CPU_SELECT(Cpu::Default))(DOXYGEN_ELLIPSIS(0)); +} +/* [Cpu-usage-automatic-cached-dispatch-compile-time] */ +} + } #endif @@ -170,6 +190,23 @@ transform(data); /* [Cpu-usage-automatic-runtime-dispatch-call] */ } +{ +using namespace Bar; +#ifndef CORRADE_CPU_USE_IFUNC +#define LOOKUP_USES_FUNCTION_POINTER +#endif +/* [Cpu-usage-automatic-cached-dispatch-call] */ +#ifdef LOOKUP_USES_FUNCTION_POINTER +int (*lookup)(DOXYGEN_ELLIPSIS(int)); +#else +int lookup(DOXYGEN_ELLIPSIS(int)); +#endif + +int found = lookup(DOXYGEN_ELLIPSIS(0)); +/* [Cpu-usage-automatic-cached-dispatch-call] */ +static_cast(found); +} + { /* [Cpu-tag-from-type] */ foo(Cpu::Avx2); diff --git a/modules/FindCorrade.cmake b/modules/FindCorrade.cmake index 714506249..b43438a0f 100644 --- a/modules/FindCorrade.cmake +++ b/modules/FindCorrade.cmake @@ -100,6 +100,8 @@ # CORRADE_TARGET_MSVC - Defined if compiling with MSVC or Clang with # a MSVC frontend # CORRADE_TARGET_MINGW - Defined if compiling under MinGW +# CORRADE_CPU_USE_IFUNC - Defined if GNU IFUNC is allowed to be used +# for runtime dispatch in the Cpu library # CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT - Defined if PluginManager # doesn't support dynamic plugin loading due to platform limitations # CORRADE_TESTSUITE_TARGET_XCTEST - Defined if TestSuite is targeting Xcode @@ -332,6 +334,7 @@ set(_corradeFlags # multi-arch binaries or when mixing different STL implementations. # TARGET_GCC etc are figured out via UseCorrade.cmake, as the compiler can # be different when compiling the lib & when using it. + CPU_USE_IFUNC PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT TESTSUITE_TARGET_XCTEST UTILITY_USE_ANSI_COLORS) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 9bd3becec..5292d4079 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -770,6 +770,20 @@ compile time (`-msimd128` passed to Clang). #define CORRADE_TARGET_SIMD128 #undef CORRADE_TARGET_SIMD128 +/** +@brief GNU IFUNC is allowed to be used for runtime dispatch in the Cpu library +@m_since_latest + +Defined if the @relativeref{Corrade,Cpu} library can perform runtime dispatch +using [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC), exposing the +@ref CORRADE_CPU_DISPATCHED_IFUNC() macro. Supported only on Linux with glibc +and on Android with API 30+. See @ref Cpu-usage-automatic-cached-dispatch for +details and information about performance tradeoffs. +@see @ref building-corrade, @ref corrade-cmake +*/ +#define CORRADE_CPU_USE_IFUNC +#undef CORRADE_CPU_USE_IFUNC + /** @brief PluginManager doesn't have dynamic plugin support on this platform diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index b78441b59..ab7aa6d12 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -299,6 +299,58 @@ usually just one or two out of the whole set: On the call side, there's no difference. The created dispatcher function takes @ref Features as well. + +@section Cpu-usage-automatic-cached-dispatch Automatic cached dispatch + +Ultimately, the dispatch can be performed implicitly, exposing only the final +function or a function pointer, with no additional steps needed from the user +side. There's three possible scenarios with varying performance tradeoffs. +Continuing from the @cpp lookupImplementation() @ce example above: + +
    +
  • +On Linux and Android with API 30+ it's possible to use the +[GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) mechanism, where the +dynamic linker performs a dispatch during the early startup. This is the +fastest variant of runtime dispatch, as it results in an equivalent of a +regular dynamic library function call. Assuming a dispatcher was created using +either @ref CORRADE_CPU_DISPATCHER() or @ref CORRADE_CPU_DISPATCHER_BASE(), +it's implemented using the @ref CORRADE_CPU_DISPATCHED_IFUNC() macro: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-ifunc +
    • +
  • +On platforms where IFUNC isn't available, a function pointer can be used +for runtime dispatch instead. It's one additional indirection, which may have a +visible effect if the dispatched-to code is relatively tiny and is called from +within a tight loop. Assuming a dispatcher was created using +either @ref CORRADE_CPU_DISPATCHER() or @ref CORRADE_CPU_DISPATCHER_BASE(), +it's implemented using the @ref CORRADE_CPU_DISPATCHED_POINTER() macro: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-pointer +
    • +
  • +For the least amount of overhead, the compile-time dispatch can be used, with +arguments passed through by hand. Similarly to IFUNC, this will also result in +a regular function, but without the indirect overhead. Furthermore, since it's +a direct call to the lambda inside, compiler optimizations will fully inline +its contents, removing any remaining overhead and allowing LTO and other +inter-procedural optimizations that wouldn't be possible with the indirect +calls. This option is best suited for scenarios where it's possible to build +and optimize code for a single target platform. In this case it calls directly +to the original variants, so no macro is needed and +@ref CORRADE_CPU_DISPATCHER() / @ref CORRADE_CPU_DISPATCHER_BASE() is not +needed either: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-compile-time +
    • +
+ +With all three cases, you end up with either a function or a function pointer. +Finally, when exposed in a header as appropriate, both variants can be then +called the same way: + +@snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-call */ namespace Cpu { @@ -1786,6 +1838,100 @@ For a dispatch using just the base instruction set use _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS( _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(type, __VA_ARGS__)) #endif +/** +@brief Create a runtime-dispatched function pointer +@m_since_latest + +Assuming a @p dispatcher of @p type was defined with either +@ref CORRADE_CPU_DISPATCHER() or @ref CORRADE_CPU_DISPATCHER_BASE(), defines a +function pointer variable of @p type and @p name. In a global constructor the +variable is assigned a function pointer returned by @p dispatcher for +@relativeref{Corrade,Cpu::runtimeFeatures()}. + +The pointer can be changed afterwards, such as for testing purposes, See also +@ref CORRADE_CPU_DISPATCHED_IFUNC() which avoids the overhead of function +pointer indirection. + +See @ref Cpu-usage-automatic-cached-dispatch for more information, usage +example and overhead comparison. +*/ +#define CORRADE_CPU_DISPATCHED_POINTER(type, dispatcher, name) \ + type name = dispatcher(Corrade::Cpu::runtimeFeatures()); + +/** +@brief Create a runtime-dispatched function via GNU IFUNC +@m_since_latest + +Available only if @ref CORRADE_CPU_USE_IFUNC is enabled. Assuming a +@p dispatcher of @p type was defined with either @ref CORRADE_CPU_DISPATCHER() +or @ref CORRADE_CPU_DISPATCHER_BASE(), defines a function with a signature +specified via the third variadic argument. The signature has to match @p type. +The function uses the [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) +mechanism, which causes the function call to be resolved to a function pointer +returned by @p dispatcher for @relativeref{Corrade,Cpu::runtimeFeatures()}. The +dispatch is performed by the dynamic linker during early startup and cannot be +changed afterwards. + +If @ref CORRADE_CPU_USE_IFUNC isn't available, is explicitly disabled or if +you need to be able to subsequently change the dispatched-to function (such as +for testing purposes), use @ref CORRADE_CPU_DISPATCHED_POINTER() instead. + +See @ref Cpu-usage-automatic-cached-dispatch for more information, usage +example and overhead comparison. +*/ +#if defined(CORRADE_CPU_USE_IFUNC) || defined(DOXYGEN_GENERATING_OUTPUT) +/* On ARM we get CPU features through getauxval() but it can't be called from + an ifunc resolver because it's too early at that point. Instead, AT_HWCAPS + is passed to it from outside, so there we call an internal variant with the + caps parameter -- see its documentation in Cpu.cpp for more info. On x86 + calling into CPUID from within an ifunc resolver is no problem. + + Although not specifically documented anywhere, the dispatcher has to have + C++ mangling disabled in order to be found by __attribute__((ifunc)), on + both GCC and Clang. That however means it's exported even if inside an + anonymous namespace, which is undesirable. To fix that, it's marked as + static... */ +#ifdef CORRADE_TARGET_CLANG +/* ... unfortunately the static makes Clang not find the name again, so there + we can't use it. But, to drown even deeper, not using the static causes the + -Wmissing-prototypes macro to get fired (which is enabled globally because + it has obvious benefits), so to avoid noise it has to be disabled here. */ +#ifndef CORRADE_TARGET_ARM +#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmissing-prototypes\"") \ + extern "C" type dispatcher() { \ + return dispatcher(Corrade::Cpu::runtimeFeatures()); \ + } \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); \ + _Pragma("GCC diagnostic pop") +#else +#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmissing-prototypes\"") \ + extern "C" type dispatcher(unsigned long caps) { \ + return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ + } \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); \ + _Pragma("GCC diagnostic pop") +#endif +#else +#ifndef CORRADE_TARGET_ARM +#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ + extern "C" { static type dispatcher() { \ + return dispatcher(Corrade::Cpu::runtimeFeatures()); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); +#else +#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ + extern "C" { static type dispatcher(unsigned long caps) { \ + return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher))); +#endif +#endif +#endif + #if defined(CORRADE_TARGET_X86) || defined(DOXYGEN_GENERATING_OUTPUT) /** @brief Enable SSE2 for given function diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index 5152a4037..844e44957 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -101,6 +101,10 @@ struct CpuTest: TestSuite::Tester { void tagDispatchRuntimeExtraCombination(); void tagDispatchRuntimeExtraZeroExtra(); + void tagDispatchedCompileTime(); + void tagDispatchedPointer(); + void tagDispatchedIfunc(); + template void enableMacros(); void debug(); @@ -155,6 +159,10 @@ CpuTest::CpuTest() { &CpuTest::tagDispatchRuntimeExtraCombination, &CpuTest::tagDispatchRuntimeExtraZeroExtra, + &CpuTest::tagDispatchedCompileTime, + &CpuTest::tagDispatchedPointer, + &CpuTest::tagDispatchedIfunc, + #ifdef CORRADE_TARGET_X86 &CpuTest::enableMacros, &CpuTest::enableMacros, @@ -1042,6 +1050,146 @@ void CpuTest::tagDispatchRuntimeExtraZeroExtra() { #endif } +using DispatchedT = Cpu::Features(*)(); + +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return []() -> Cpu::Features { return Cpu::Scalar; }; +} +#ifdef CORRADE_TARGET_X86 +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2)) { + return []() -> Cpu::Features { return Cpu::Sse2; }; +} +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Avx)) { + return []() -> Cpu::Features { return Cpu::Avx; }; +} +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Avx|Cpu::AvxFma)) { + return []() -> Cpu::Features { return Cpu::Avx|Cpu::AvxFma; }; +} + +CORRADE_CPU_DISPATCHER(DispatchedT, dispatchedImplementation, Cpu::AvxFma) +#elif defined(CORRADE_TARGET_ARM) +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::NeonFp16)) { + return []() -> Cpu::Features { return Cpu::NeonFp16; }; +} +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Neon)) { + return []() -> Cpu::Features { return Cpu::Neon; }; +} + +CORRADE_CPU_DISPATCHER(DispatchedT, dispatchedImplementation) +#elif defined(CORRADE_TARGET_WASM) +CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Simd128)) { + return []() -> Cpu::Features { return Cpu::Simd128; }; +} + +CORRADE_CPU_DISPATCHER(DispatchedT, dispatchedImplementation) +#endif + +/* CORRADE_NEVER_INLINE to make it possible to look at the disassembly. The + lambda body should be fully inlined here. */ +CORRADE_NEVER_INLINE Cpu::Features dispatchedCompileTime() { + /* While calling without CORRADE_CPU_SELECT() would work too, it'd go + through the runtime dispatch. And then the call would *not* get inlined, + no matter how many force inlines I put onto the lambdas. */ + return dispatchedImplementation(CORRADE_CPU_SELECT(Cpu::Default))(); +} + +void CpuTest::tagDispatchedCompileTime() { + Cpu::Features dispatchedFeatures = dispatchedCompileTime(); + CORRADE_INFO("Dispatched to:" << dispatchedFeatures); + + Cpu::Features features = Cpu::compiledFeatures(); + #ifdef CORRADE_TARGET_X86 + if(features >= (Cpu::Avx|Cpu::AvxFma)) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx|Cpu::AvxFma); + else if(features >= Cpu::Avx) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx); + else if(features >= Cpu::Sse2) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Sse2); + else + #elif defined(CORRADE_TARGET_ARM) + if(features >= Cpu::NeonFp16) + CORRADE_COMPARE(dispatchedFeatures, Cpu::NeonFp16); + else if(features >= Cpu::Neon) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Neon); + else + #elif defined(CORRADE_TARGET_WASM) + if(features >= Cpu::Simd128) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Simd128); + else + #endif + { + CORRADE_COMPARE(dispatchedFeatures, Cpu::Scalar); + } +} + +CORRADE_CPU_DISPATCHED_POINTER(DispatchedT, dispatchedImplementation, dispatchedPointer) + +void CpuTest::tagDispatchedPointer() { + Cpu::Features dispatchedFeatures = dispatchedPointer(); + CORRADE_INFO("Dispatched to:" << dispatchedFeatures); + + Cpu::Features features = Cpu::runtimeFeatures(); + #ifdef CORRADE_TARGET_X86 + if(features >= (Cpu::Avx|Cpu::AvxFma)) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx|Cpu::AvxFma); + else if(features >= Cpu::Avx) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx); + else if(features >= Cpu::Sse2) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Sse2); + else + #elif defined(CORRADE_TARGET_ARM) + if(features >= Cpu::NeonFp16) + CORRADE_COMPARE(dispatchedFeatures, Cpu::NeonFp16); + else if(features >= Cpu::Neon) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Neon); + else + #elif defined(CORRADE_TARGET_WASM) + if(features >= Cpu::Simd128) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Simd128); + else + #endif + { + CORRADE_COMPARE(dispatchedFeatures, Cpu::Scalar); + } +} + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_DISPATCHED_IFUNC(DispatchedT, dispatchedImplementation, Cpu::Features dispatchedIfunc()) +#endif + +void CpuTest::tagDispatchedIfunc() { + #ifndef CORRADE_CPU_USE_IFUNC + CORRADE_SKIP("CORRADE_CPU_USE_IFUNC not available"); + #else + Cpu::Features dispatchedFeatures = dispatchedIfunc(); + CORRADE_INFO("Dispatched to:" << dispatchedFeatures); + + Cpu::Features features = Cpu::runtimeFeatures(); + #ifdef CORRADE_TARGET_X86 + if(features >= (Cpu::Avx|Cpu::AvxFma)) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx|Cpu::AvxFma); + else if(features >= Cpu::Avx) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Avx); + else if(features >= Cpu::Sse2) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Sse2); + else + #elif defined(CORRADE_TARGET_ARM) + if(features >= Cpu::NeonFp16) + CORRADE_COMPARE(dispatchedFeatures, Cpu::NeonFp16); + else if(features >= Cpu::Neon) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Neon); + else + #elif defined(CORRADE_TARGET_WASM) + if(features >= Cpu::Simd128) + CORRADE_COMPARE(dispatchedFeatures, Cpu::Simd128); + else + #endif + { + CORRADE_COMPARE(dispatchedFeatures, Cpu::Scalar); + } + #endif +} + /* Not using an argument here since we *don't* want the overload delegating in this case -- it would hide errors when a certain instruction set doesn't have a corresponding overload, as it'd fall back to a parent one. I'm also diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index e22dd75ee..59794e6ac 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -44,6 +44,7 @@ #cmakedefine CORRADE_TARGET_EMSCRIPTEN #cmakedefine CORRADE_TARGET_ANDROID +#cmakedefine CORRADE_CPU_USE_IFUNC #cmakedefine CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT #cmakedefine CORRADE_TESTSUITE_TARGET_XCTEST #cmakedefine CORRADE_UTILITY_USE_ANSI_COLORS From a03fa513c0100462fbf898f95ade8a680520949c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 17 Jul 2022 22:06:21 +0200 Subject: [PATCH 25/79] Cpu: print also unknown bits in the Features debug printer. Because yes this -- random crap -- is what I get when I try to use IFUNC on GCC 4.8. --- src/Corrade/Cpu.cpp | 12 ++++++++++-- src/Corrade/Test/CpuTest.cpp | 24 ++++++++++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index ea45aa236..63311d05a 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -313,8 +313,10 @@ Features runtimeFeatures() { } #endif -Utility::Debug& operator<<(Utility::Debug& debug, const Features value) { - const Containers::StringView prefix = "|Cpu::"_s.exceptSuffix(debug.immediateFlags() & Utility::Debug::Flag::Packed ? 5 : 0); +Utility::Debug& operator<<(Utility::Debug& debug, Features value) { + const bool packed = debug.immediateFlags() >= Utility::Debug::Flag::Packed; + + const Containers::StringView prefix = "|Cpu::"_s.exceptSuffix(packed ? 5 : 0); /* First one without the | */ debug << prefix.exceptPrefix(1) << Utility::Debug::nospace; @@ -326,6 +328,7 @@ Utility::Debug& operator<<(Utility::Debug& debug, const Features value) { if(!written) written = true; \ else debug << Utility::Debug::nospace << prefix << Utility::Debug::nospace; \ debug << TypeTraits::name(); \ + value &= ~tag; \ } #ifdef CORRADE_TARGET_X86 _c(Sse2) @@ -351,6 +354,11 @@ Utility::Debug& operator<<(Utility::Debug& debug, const Features value) { #endif #undef _c + if(value) { + if(written) debug << Utility::Debug::nospace << prefix << Utility::Debug::nospace; + debug << (packed ? "" : "Features(") << Utility::Debug::nospace << reinterpret_cast(static_cast(value)) << Utility::Debug::nospace << (packed ? "" : ")"); + } + return debug; } diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index 844e44957..629a3122a 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -1487,31 +1487,39 @@ template void CpuTest::enableMacros() { } void CpuTest::debug() { + /* If more flags get added, this might need to become even more zeros */ + const unsigned int dead = 0xde00ad00; + /* Features{} are equivalent to Scalar */ #ifdef CORRADE_TARGET_X86 std::ostringstream out; - Debug{&out} << Cpu::Scalar << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Cpu::Features{}; - CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Ssse3|Cpu::Sse41|Cpu::Avx2 Cpu::Scalar\n"); + Debug{&out} << Cpu::Scalar << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Cpu::Features{} << (reinterpret_cast(dead)|Cpu::Avx512f) << reinterpret_cast(dead); + CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Ssse3|Cpu::Sse41|Cpu::Avx2 Cpu::Scalar Cpu::Avx512f|Cpu::Features(0xde00ad00) Cpu::Features(0xde00ad00)\n"); #elif defined(CORRADE_TARGET_ARM) std::ostringstream out; - Debug{&out} << Cpu::Scalar << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Cpu::Features{}; - CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Neon|Cpu::NeonFma|Cpu::NeonFp16 Cpu::Scalar\n"); + Debug{&out} << Cpu::Scalar << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Cpu::Features{} << (reinterpret_cast(dead)|Cpu::NeonFma) << reinterpret_cast(dead); + CORRADE_COMPARE(out.str(), "Cpu::Scalar Cpu::Neon|Cpu::NeonFma|Cpu::NeonFp16 Cpu::Scalar Cpu::NeonFma|Cpu::Features(0xde00ad00) Cpu::Features(0xde00ad00)\n"); #else + static_cast(dead); CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif } void CpuTest::debugPacked() { + /* If more flags get added, this might need to become even more zeros */ + const unsigned int dead = 0xde00ad00; + /* Features{} are equivalent to Scalar */ #ifdef CORRADE_TARGET_X86 std::ostringstream out; - Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Debug::packed << Cpu::Features{} << Cpu::Avx; - CORRADE_COMPARE(out.str(), "Scalar Ssse3|Sse41|Avx2 Scalar Cpu::Avx\n"); + Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::Avx2|Cpu::Ssse3|Cpu::Sse41) << Debug::packed << Cpu::Features{} << Debug::packed << (reinterpret_cast(dead)|Cpu::Avx512f) << Debug::packed << reinterpret_cast(dead) << Cpu::Avx; + CORRADE_COMPARE(out.str(), "Scalar Ssse3|Sse41|Avx2 Scalar Avx512f|0xde00ad00 0xde00ad00 Cpu::Avx\n"); #elif defined(CORRADE_TARGET_ARM) std::ostringstream out; - Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Debug::packed << Cpu::Features{} << Cpu::NeonFma; - CORRADE_COMPARE(out.str(), "Scalar Neon|NeonFma|NeonFp16 Scalar Cpu::NeonFma\n"); + Debug{&out} << Debug::packed << Cpu::Scalar << Debug::packed << (Cpu::NeonFp16|Cpu::NeonFma|Cpu::Neon) << Debug::packed << Cpu::Features{} << Debug::packed << (reinterpret_cast(dead)|Cpu::NeonFma) << Debug::packed << reinterpret_cast(dead) << Cpu::NeonFma; + CORRADE_COMPARE(out.str(), "Scalar Neon|NeonFma|NeonFp16 Scalar NeonFma|0xde00ad00 0xde00ad00 Cpu::NeonFma\n"); #else + static_cast(dead); CORRADE_SKIP("Not enough Cpu tags available on this platform, can't test"); #endif } From 0662dbee8481a88906095ede9add913f6029641f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 18 Jul 2022 12:00:10 +0200 Subject: [PATCH 26/79] Cpu: make ifunc work on GCC 4.8. This just applies the patch that went into GCC 4.9 but to our code. --- src/Corrade/Cpu.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index ab7aa6d12..48cf63f9b 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1915,7 +1915,29 @@ example and overhead comparison. __VA_ARGS__ __attribute__((ifunc(#dispatcher))); \ _Pragma("GCC diagnostic pop") #endif +#elif defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ < 409 +/* Furthermore, due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58105 + the resolver function won't work on GCC 4.8 unless it's marked with + CORRADE_NEVER_INLINE. That, however, causes GCC 4.8 to spit out a bogus + warning that the dispatcher function is redeclared as noinline -- it thinks + it's the same function as the always-inline lambda wrappers which have the + same name. Despite the warning it works, but to avoid useless noise the + ifunc dispatcher is named differently. */ +#ifndef CORRADE_TARGET_ARM +#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ + extern "C" { CORRADE_NEVER_INLINE static type dispatcher ## Ifunc() { \ + return dispatcher(Corrade::Cpu::runtimeFeatures()); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher "Ifunc"))); +#else +#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ + extern "C" { CORRADE_NEVER_INLINE static type dispatcher ## Ifunc(unsigned long caps) { \ + return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ + }} \ + __VA_ARGS__ __attribute__((ifunc(#dispatcher "Ifunc"))); +#endif #else +/* Only GCC 4.9+ has the implementation in the most minimal form. */ #ifndef CORRADE_TARGET_ARM #define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ extern "C" { static type dispatcher() { \ From eccc8cb61a45631082e7a460914f730f5c0a1c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 18 Jul 2022 13:22:07 +0200 Subject: [PATCH 27/79] Cpu: disable IFUNC on GCC 4.8 if --coverage is used. The offending flag is -fprofile-arcs and it's failing only on Ubuntu 20.04 with GCC 4.8, not with GCC 5 and not with GCC 4.8 on Arch. I couldn't find any even vaguely related bugreport, so I suspect this has something to with binutils or something else on that system. But since GCC 5 works there, I don't see that much of a problem. Unfortunately I also couldn't find any way to detect if such flag is present, the predefined variables are exactly the same in both cases. Which means this won't be able to detect if Corrade is built without --coverage and then used in code that has --coverage enabled. But as I'm probably the only person using --converage on GCC 4.8 (on the CI), I don't think that's a problem either. --- CMakeLists.txt | 21 ++++++++++++++++++++- doc/building-corrade.dox | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f353b0748..8a9dbfd24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -206,10 +206,29 @@ extern \"C\" int(*fooDispatcher())() { int foo() __attribute__((ifunc(\"fooDispatcher\"))); int main() { return foo() - 42; }\ " _CORRADE_CPU_CAN_USE_IFUNC) + # In cases where ifunc is known to be broken, disable it by default -- + # users can still force it to be enabled, but the initial state should not + # cause crashes or other strange behavior. + if(_CORRADE_CPU_CAN_USE_IFUNC) + set(_CORRADE_CPU_USE_IFUNC_DEFAULT ON) + # On GCC 4.8, if --coverage or -fprofile-arcs is enabled, the ifunc + # dispatchers cause a segfault. On Ubuntu 20.04 at least. Not the case + # with GCC 5 there, not the case with GCC 4.8 on Arch. Can't find any + # upstream bug report or commit that would be related to this. + if(CMAKE_CXX_COMPILER_ID AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9" AND CMAKE_CXX_FLAGS MATCHES "(--coverage|-fprofile-arcs)") + if(NOT DEFINED CORRADE_CPU_USE_IFUNC) + message(WARNING "Disabling CORRADE_CPU_USE_IFUNC by default as it may crash when used together with --coverage on GCC 4.8.") + endif() + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) + endif() + else() + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) + endif() else() set(_CORRADE_CPU_CAN_USE_IFUNC OFF) + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) endif() -cmake_dependent_option(CORRADE_CPU_USE_IFUNC "Allow using GNU IFUNC for runtime CPU dispatch" ON "_CORRADE_CPU_CAN_USE_IFUNC" OFF) +cmake_dependent_option(CORRADE_CPU_USE_IFUNC "Allow using GNU IFUNC for runtime CPU dispatch" ${_CORRADE_CPU_USE_IFUNC_DEFAULT} "_CORRADE_CPU_CAN_USE_IFUNC" OFF) # Backwards compatibility for unprefixed CMake options. If the user isn't # explicitly using prefixed options in the first run already, accept the diff --git a/doc/building-corrade.dox b/doc/building-corrade.dox index 32fb32d9c..a94974000 100644 --- a/doc/building-corrade.dox +++ b/doc/building-corrade.dox @@ -557,7 +557,8 @@ Platform-specific options: [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) to be used for runtime dispatch in the @ref Cpu library. Available only on platforms that support it, which is currently only Linux with glibc and Android with API - 30+, and there enabled by default. See + 30+, and there enabled by default unless a problematic case is detected + that may cause it to misbehave. See @ref Cpu-usage-automatic-cached-dispatch for details and information about performance tradeoffs. - `CORRADE_UTILITY_USE_ANSI_COLORS` --- if building for Windows, this will From 10ccf25ac7631d69a599a11dbc23d7e5cb9e6c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 18 Jul 2022 15:49:20 +0200 Subject: [PATCH 28/79] Cpu: disable IFUNC with sanitizers, and fail if it's enabled on them. Since Corrade can be built without -fsanitize= first and then used by code that has -fsanitize= enabled, there needs to be also a check directly inside configure.h. --- CMakeLists.txt | 22 ++++++++++++++++++++++ src/Corrade/configure.h.cmake | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a9dbfd24..f9399774d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,6 +221,28 @@ int main() { return foo() - 42; }\ endif() set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) endif() + + # If sanitizers are enabled, call into the dispatch function crashes. + # Upstream bugreport https://github.com/google/sanitizers/issues/342 + # suggests using __attribute__((no_sanitize_address)), but that doesn't + # work / can't be used because it would mean marking basically + # everything including the actual implementation that's being + # dispatched to. + # + # Sanitizers can be also enabled any other way such as with + # CMAKE_XCODE_SCHEME_ADDRESS_SANITIZER or with per-target properties + # which isn't easy to detect, so as a secondary measure there's a kill + # switch in Corrade/configure.h that will fail with an error if a + # sanitizer is detected and CORRADE_CPU_USE_IFUNC is enabled. + # + # While Clang and GCC use -fsanitize=whatever, MSVC allows also + # /fsanitize=, so catch both. + if(CMAKE_CXX_FLAGS MATCHES "[-/]fsanitize=") + if(NOT DEFINED CORRADE_CPU_USE_IFUNC) + message(WARNING "Disabling CORRADE_CPU_USE_IFUNC by default as it crashes when used together with sanitizers. See https://github.com/google/sanitizers/issues/342 for more information.") + endif() + set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) + endif() else() set(_CORRADE_CPU_USE_IFUNC_DEFAULT OFF) endif() diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 59794e6ac..686ca8980 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -361,4 +361,23 @@ static_assert(sizeof(1 ? "" : "") == 1, "enabled or ensure /permissive- is set for all files that include Corrade " "headers."); #endif +/* Kill switch for when presence of a sanitizer is detected and + CORRADE_CPU_USE_IFUNC is enabled. Unfortunately in our case the + __attribute__((no_sanitize_address)) workaround as described on + https://github.com/google/sanitizers/issues/342 doesn't work / can't be used + because it would mean marking basically everything including the actual + implementation that's being dispatched to. */ +#ifdef CORRADE_CPU_USE_IFUNC +#ifdef __has_feature +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || __has_feature(memory_sanitizer) || __has_feature(undefined_behavior_sanitizer) +#define _CORRADE_SANITIZER_IFUNC_DETECTED +#endif +#elif defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) +#define _CORRADE_SANITIZER_IFUNC_DETECTED +#endif +#ifdef _CORRADE_SANITIZER_IFUNC_DETECTED +#error Corrade was built with CORRADE_CPU_USE_IFUNC, which is incompatible with sanitizers. Rebuild without this option or disable sanitizers. +#endif +#endif + #endif // kate: hl c++ From b8a054f0343c9b883deda93135cee0ebf883381a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 18 Jul 2022 20:42:41 +0200 Subject: [PATCH 29/79] Add a placeholder CORRADE_BUILD_CPU_RUNTIME_DISPATCH option. To be used by code that actually provides multiple variants. Enabled by default only on platforms that have IFUNC support, elsewhere it's currently disabled because I'm not yet sure about perf consequences of going through a function pointer. --- CMakeLists.txt | 5 +++++ doc/building-corrade.dox | 11 +++++++++++ doc/corrade-changelog.dox | 7 +++++++ doc/corrade-cmake.dox | 3 +++ modules/FindCorrade.cmake | 4 ++++ src/Corrade/Corrade.h | 19 +++++++++++++++++++ src/Corrade/configure.h.cmake | 1 + 7 files changed, 50 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f9399774d..c2f55c8d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -252,6 +252,11 @@ else() endif() cmake_dependent_option(CORRADE_CPU_USE_IFUNC "Allow using GNU IFUNC for runtime CPU dispatch" ${_CORRADE_CPU_USE_IFUNC_DEFAULT} "_CORRADE_CPU_CAN_USE_IFUNC" OFF) +# Runtime CPU dispatch. Because going through a function pointer may have +# negative perf consequences, enable it by default only on platforms that have +# IFUNC, and thus can avoid the function pointer indirection. +option(CORRADE_BUILD_CPU_RUNTIME_DISPATCH "Build with runtime dispatch for CPU-dependent functionality" ${_CORRADE_CPU_CAN_USE_IFUNC}) + # Backwards compatibility for unprefixed CMake options. If the user isn't # explicitly using prefixed options in the first run already, accept the # unprefixed options, and remember this decision for subsequent runs diff --git a/doc/building-corrade.dox b/doc/building-corrade.dox index a94974000..53457bd17 100644 --- a/doc/building-corrade.dox +++ b/doc/building-corrade.dox @@ -550,6 +550,17 @@ Options controlling the build: Corrade features simultaneously in multiple threads. Enabled by default, disable if you don't need this and don't want to pay potential performance penalties coming from thread-local variables. +- `CORRADE_BUILD_CPU_RUNTIME_DISPATCH` --- Build with performance-critical + code paths optimized for multiple architectures (such as SSE or AVX on + x86), with the best matching variant selected at runtime based on detected + CPU features. If not enabled, the library is built with just a single + variant that's picked at compile time depending on target architecture + flags being passed to the compiler. Enabled by default on platforms that + support [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC), which is + currently only Linux with glibc and Android with API 30+. See also the + `CORRADE_CPU_USE_IFUNC` option below and + @ref Cpu-usage-automatic-cached-dispatch for details and information about + performance tradeoffs. Platform-specific options: diff --git a/doc/corrade-changelog.dox b/doc/corrade-changelog.dox index 584ac8268..d707dfd99 100644 --- a/doc/corrade-changelog.dox +++ b/doc/corrade-changelog.dox @@ -364,6 +364,13 @@ namespace Corrade { Windows as well instead of declaring aligned memory allocation functions on its own, as it's not worth the seriously-looking compiler warnings (see [mosra/corrade#145](https://github.com/mosra/corrade/issues/145)) +- Platforms that support @ref CORRADE_CPU_USE_IFUNC will now build certain + code paths optimized for multiple architectures, with the best variant + selected at runtime using the @ref Cpu library based on available CPU + features. This behavior can be disabled with the + @ref CORRADE_BUILD_CPU_RUNTIME_DISPATCH CMake option. Platforms without + IFUNC support implement runtime dispatch using function pointers instead of + indirect functions and have this currently disabled by default. @subsection corrade-changelog-latest-bugfixes Bug fixes diff --git a/doc/corrade-cmake.dox b/doc/corrade-cmake.dox index 27ed7da6d..8b87b25ba 100644 --- a/doc/corrade-cmake.dox +++ b/doc/corrade-cmake.dox @@ -211,6 +211,9 @@ also available as preprocessor variables if you include - `CORRADE_BUILD_MULTITHREADED` --- Defined if compiled in a way that makes it possible to safely use certain Corrade features simultaneously in multiple threads. +- `CORRADE_BUILD_CPU_RUNTIME_DISPATCH` --- Defined if built with code paths + optimized for multiple architectres with the best matching variant selected + at runtime based on detected CPU features - `CORRADE_TARGET_UNIX` --- Defined if compiled for some Unix flavor (Linux, BSD, macOS, iOS, Android, ...) - `CORRADE_TARGET_APPLE` --- Defined if compiled for Apple platforms diff --git a/modules/FindCorrade.cmake b/modules/FindCorrade.cmake index b43438a0f..1343f3feb 100644 --- a/modules/FindCorrade.cmake +++ b/modules/FindCorrade.cmake @@ -80,6 +80,9 @@ # CORRADE_BUILD_MULTITHREADED - Defined if compiled in a way that makes it # possible to safely use certain Corrade features simultaneously in multiple # threads +# CORRADE_BUILD_CPU_RUNTIME_DISPATCH - Defined if built with code paths +# optimized for multiple architectres with the best matching variant selected +# at runtime based on detected CPU features # CORRADE_TARGET_UNIX - Defined if compiled for some Unix flavor # (Linux, BSD, macOS) # CORRADE_TARGET_APPLE - Defined if compiled for Apple platforms @@ -321,6 +324,7 @@ set(_corradeFlags BUILD_STATIC BUILD_STATIC_UNIQUE_GLOBALS BUILD_MULTITHREADED + BUILD_CPU_RUNTIME_DISPATCH TARGET_UNIX TARGET_APPLE TARGET_IOS diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 5292d4079..0a1f015b6 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -115,6 +115,25 @@ read/write access to global data. #define CORRADE_BUILD_MULTITHREADED #undef CORRADE_BUILD_MULTITHREADED +/** +@brief Build with runtime CPU dispatch +@m_since_latest + +Defined if the library is built with performance-critical code paths optimized +for multiple architectures (such as SSE or AVX on x86), with the best matching +variant selected at runtime based on detected CPU features. If not defined, the +library is built with just a single variant that's picked at compile time +depending on target architecture flags being passed to the compiler. + +The actual feature detection and dispatch both in the runtime and compile-time +scenario is performed by the @relativeref{Corrade,Cpu} library. See +@ref Cpu-usage-automatic-cached-dispatch for details and information about +performance tradeoffs. +@see @see @ref CORRADE_CPU_USE_IFUNC, @ref building-corrade, @ref corrade-cmake +*/ +#define CORRADE_BUILD_CPU_RUNTIME_DISPATCH +#undef CORRADE_BUILD_CPU_RUNTIME_DISPATCH + /** @brief Debug build diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 686ca8980..0445d9056 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -34,6 +34,7 @@ #cmakedefine CORRADE_BUILD_STATIC #cmakedefine CORRADE_BUILD_STATIC_UNIQUE_GLOBALS #cmakedefine CORRADE_BUILD_MULTITHREADED +#cmakedefine CORRADE_BUILD_CPU_RUNTIME_DISPATCH #cmakedefine CORRADE_TARGET_APPLE #cmakedefine CORRADE_TARGET_IOS From 2847c9b44d37034e1c6f6cf667582e7a08507d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Jul 2022 11:33:11 +0200 Subject: [PATCH 30/79] Forward-declare Cpu::Features in Corrade.h. So we can expose internal dispatcher functions without having to pull in the full header. --- src/Corrade/Corrade.h | 8 +++++++- src/Corrade/Cpu.h | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 0a1f015b6..b35989038 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -27,7 +27,7 @@ */ /** @file - * @brief Basic definitions + * @brief Basic definitions and forward declarations for the @ref Corrade namespace */ #include "Corrade/configure.h" @@ -844,6 +844,12 @@ the console. This is done automatically when you link to the #undef CORRADE_UTILITY_USE_ANSI_COLORS #endif +#ifndef DOXYGEN_GENERATING_OUTPUT +namespace Cpu { + class Features; +} +#endif + } #endif diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 48cf63f9b..eb0e4f3ef 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -31,6 +31,7 @@ * @m_since_latest */ +#include "Corrade/Corrade.h" #include "Corrade/Utility/Macros.h" #include "Corrade/Utility/Utility.h" #include "Corrade/Utility/visibility.h" @@ -1232,8 +1233,6 @@ Returns a tag corresponding to tag type @p T. The following two expressions are template constexpr T tag() { return T{Implementation::Init}; } #if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) -class Features; - namespace Implementation { CORRADE_UTILITY_EXPORT Features runtimeFeatures(unsigned long caps); } From d55c09a858c107db6bd01a0a0255fdba64c625e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Jul 2022 11:58:19 +0200 Subject: [PATCH 31/79] Cpu: unify CORRADE_CPU_DISPATCHED_POINTER() and _IFUNC() signatures. This way it's much easier to make a wrapper macro that delegates to one or the other. --- doc/snippets/Corrade.cpp | 2 +- src/Corrade/Cpu.h | 12 ++++++------ src/Corrade/Test/CpuTest.cpp | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index 3b03740a9..1961ff727 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -129,7 +129,7 @@ CORRADE_CPU_DISPATCHED_IFUNC(LookupT, lookupImplementation, int lookup(DOXYGEN_E /* [Cpu-usage-automatic-cached-dispatch-ifunc] */ #else /* [Cpu-usage-automatic-cached-dispatch-pointer] */ -CORRADE_CPU_DISPATCHED_POINTER(LookupT, lookupImplementation, lookup) +CORRADE_CPU_DISPATCHED_POINTER(lookupImplementation, int(*lookup)(DOXYGEN_ELLIPSIS(int))) /* [Cpu-usage-automatic-cached-dispatch-pointer] */ #endif diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index eb0e4f3ef..3f5ab9217 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1841,10 +1841,10 @@ For a dispatch using just the base instruction set use @brief Create a runtime-dispatched function pointer @m_since_latest -Assuming a @p dispatcher of @p type was defined with either -@ref CORRADE_CPU_DISPATCHER() or @ref CORRADE_CPU_DISPATCHER_BASE(), defines a -function pointer variable of @p type and @p name. In a global constructor the -variable is assigned a function pointer returned by @p dispatcher for +Assuming a @p dispatcher was defined with either @ref CORRADE_CPU_DISPATCHER() +or @ref CORRADE_CPU_DISPATCHER_BASE(), defines a function pointer variable with +a signature specified in the second variadic argument. In a global constructor +the variable is assigned a function pointer returned by @p dispatcher for @relativeref{Corrade,Cpu::runtimeFeatures()}. The pointer can be changed afterwards, such as for testing purposes, See also @@ -1854,8 +1854,8 @@ pointer indirection. See @ref Cpu-usage-automatic-cached-dispatch for more information, usage example and overhead comparison. */ -#define CORRADE_CPU_DISPATCHED_POINTER(type, dispatcher, name) \ - type name = dispatcher(Corrade::Cpu::runtimeFeatures()); +#define CORRADE_CPU_DISPATCHED_POINTER(dispatcher, ...) \ + __VA_ARGS__ = dispatcher(Corrade::Cpu::runtimeFeatures()); /** @brief Create a runtime-dispatched function via GNU IFUNC diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index 629a3122a..e9e3a21a8 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -1122,7 +1122,7 @@ void CpuTest::tagDispatchedCompileTime() { } } -CORRADE_CPU_DISPATCHED_POINTER(DispatchedT, dispatchedImplementation, dispatchedPointer) +CORRADE_CPU_DISPATCHED_POINTER(dispatchedImplementation, Cpu::Features (*dispatchedPointer)()) void CpuTest::tagDispatchedPointer() { Cpu::Features dispatchedFeatures = dispatchedPointer(); From ee6f239001bf5cfdebb7d169ca337bf521a6696a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Jul 2022 12:00:26 +0200 Subject: [PATCH 32/79] Cpu: we don't need the type in CORRADE_CPU_DISPATCHED_IFUNC(). Yay, now it's truly similar to CORRADE_CPU_DISPATCHED_POINTER(), and much less redundant. This is the kind of thing that becomes obvious only during practical use. --- doc/snippets/Corrade.cpp | 2 +- src/Corrade/Cpu.h | 36 ++++++++++++++++++++---------------- src/Corrade/Test/CpuTest.cpp | 2 +- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index 1961ff727..eba835bee 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -125,7 +125,7 @@ CORRADE_CPU_DISPATCHER(LookupT, lookupImplementation, Cpu::Popcnt, Cpu::Lzcnt) #ifdef CORRADE_CPU_USE_IFUNC /* [Cpu-usage-automatic-cached-dispatch-ifunc] */ -CORRADE_CPU_DISPATCHED_IFUNC(LookupT, lookupImplementation, int lookup(DOXYGEN_ELLIPSIS(int))) +CORRADE_CPU_DISPATCHED_IFUNC(lookupImplementation, int lookup(DOXYGEN_ELLIPSIS(int))) /* [Cpu-usage-automatic-cached-dispatch-ifunc] */ #else /* [Cpu-usage-automatic-cached-dispatch-pointer] */ diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 3f5ab9217..b0f7a6238 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -348,8 +348,12 @@ needed either: With all three cases, you end up with either a function or a function pointer. -Finally, when exposed in a header as appropriate, both variants can be then -called the same way: +The macro signatures are deliberately similar to each other and to the direct +function declaration to make it possible to unify them under a single wrapper +macro in case a practical use case needs to handle more than one variant. + +Finally, when exposed in a header as appropriate, both the function and the +function pointer variant can be then called the same way: @snippet Corrade.cpp Cpu-usage-automatic-cached-dispatch-call */ @@ -1862,8 +1866,8 @@ example and overhead comparison. @m_since_latest Available only if @ref CORRADE_CPU_USE_IFUNC is enabled. Assuming a -@p dispatcher of @p type was defined with either @ref CORRADE_CPU_DISPATCHER() -or @ref CORRADE_CPU_DISPATCHER_BASE(), defines a function with a signature +@p dispatcher was defined with either @ref CORRADE_CPU_DISPATCHER() or +@ref CORRADE_CPU_DISPATCHER_BASE(), defines a function with a signature specified via the third variadic argument. The signature has to match @p type. The function uses the [GNU IFUNC](https://sourceware.org/glibc/wiki/GNU_IFUNC) mechanism, which causes the function call to be resolved to a function pointer @@ -1896,19 +1900,19 @@ example and overhead comparison. -Wmissing-prototypes macro to get fired (which is enabled globally because it has obvious benefits), so to avoid noise it has to be disabled here. */ #ifndef CORRADE_TARGET_ARM -#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wmissing-prototypes\"") \ - extern "C" type dispatcher() { \ + extern "C" decltype(dispatcher(std::declval())) dispatcher() { \ return dispatcher(Corrade::Cpu::runtimeFeatures()); \ } \ __VA_ARGS__ __attribute__((ifunc(#dispatcher))); \ _Pragma("GCC diagnostic pop") #else -#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wmissing-prototypes\"") \ - extern "C" type dispatcher(unsigned long caps) { \ + extern "C" decltype(dispatcher(std::declval())) dispatcher(unsigned long caps) { \ return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ } \ __VA_ARGS__ __attribute__((ifunc(#dispatcher))); \ @@ -1923,14 +1927,14 @@ example and overhead comparison. same name. Despite the warning it works, but to avoid useless noise the ifunc dispatcher is named differently. */ #ifndef CORRADE_TARGET_ARM -#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ - extern "C" { CORRADE_NEVER_INLINE static type dispatcher ## Ifunc() { \ +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { CORRADE_NEVER_INLINE static decltype(dispatcher(std::declval())) dispatcher ## Ifunc() { \ return dispatcher(Corrade::Cpu::runtimeFeatures()); \ }} \ __VA_ARGS__ __attribute__((ifunc(#dispatcher "Ifunc"))); #else -#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ - extern "C" { CORRADE_NEVER_INLINE static type dispatcher ## Ifunc(unsigned long caps) { \ +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { CORRADE_NEVER_INLINE static decltype(dispatcher(std::declval())) dispatcher ## Ifunc(unsigned long caps) { \ return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ }} \ __VA_ARGS__ __attribute__((ifunc(#dispatcher "Ifunc"))); @@ -1938,14 +1942,14 @@ example and overhead comparison. #else /* Only GCC 4.9+ has the implementation in the most minimal form. */ #ifndef CORRADE_TARGET_ARM -#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ - extern "C" { static type dispatcher() { \ +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { static decltype(dispatcher(std::declval())) dispatcher() { \ return dispatcher(Corrade::Cpu::runtimeFeatures()); \ }} \ __VA_ARGS__ __attribute__((ifunc(#dispatcher))); #else -#define CORRADE_CPU_DISPATCHED_IFUNC(type, dispatcher, ...) \ - extern "C" { static type dispatcher(unsigned long caps) { \ +#define CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, ...) \ + extern "C" { static decltype(dispatcher(std::declval())) dispatcher(unsigned long caps) { \ return dispatcher(Corrade::Cpu::Implementation::runtimeFeatures(caps)); \ }} \ __VA_ARGS__ __attribute__((ifunc(#dispatcher))); diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index e9e3a21a8..fe11dd5b0 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -1154,7 +1154,7 @@ void CpuTest::tagDispatchedPointer() { } #ifdef CORRADE_CPU_USE_IFUNC -CORRADE_CPU_DISPATCHED_IFUNC(DispatchedT, dispatchedImplementation, Cpu::Features dispatchedIfunc()) +CORRADE_CPU_DISPATCHED_IFUNC(dispatchedImplementation, Cpu::Features dispatchedIfunc()) #endif void CpuTest::tagDispatchedIfunc() { From 1e8514e1f34e1ec78e33cd45f4783cc65e746468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Jul 2022 12:42:04 +0200 Subject: [PATCH 33/79] Cpu: actually I don't need the type for CORRADE_CPU_DISPATCHER*() either. Yay! This also could make it nicer to use with C++14, I hope, while still being C++11-compatible. --- doc/snippets/Corrade.cpp | 4 +-- src/Corrade/Cpu.h | 66 ++++++++++++++++++------------------ src/Corrade/Test/CpuTest.cpp | 18 +++++----- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index eba835bee..36c1c862d 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -99,7 +99,7 @@ TransformT transformImplementation(Cpu::Avx2T) { return [](Containers::ArrayView data) { DOXYGEN_ELLIPSIS(static_cast(data);) }; } -CORRADE_CPU_DISPATCHER_BASE(TransformT, transformImplementation) +CORRADE_CPU_DISPATCHER_BASE(transformImplementation) /* [Cpu-usage-automatic-runtime-dispatch-declare] */ using LookupT = int(*)(int); @@ -120,7 +120,7 @@ LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzc DOXYGEN_ELLIPSIS(return {};) } -CORRADE_CPU_DISPATCHER(LookupT, lookupImplementation, Cpu::Popcnt, Cpu::Lzcnt) +CORRADE_CPU_DISPATCHER(lookupImplementation, Cpu::Popcnt, Cpu::Lzcnt) /* [Cpu-usage-automatic-runtime-dispatch-extra-declare] */ #ifdef CORRADE_CPU_USE_IFUNC diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index b0f7a6238..7fcee3d84 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1668,12 +1668,12 @@ best viable overload. @m_since_latest Given a set of function overloads named @p function that accept a CPU tag as a -parameter and return a function pointer of @p type, creates a function with -signature @cpp type function(Cpu::Features) @ce which will select among the -overloads using a runtime-specified @relativeref{Corrade,Cpu::Features}, using -the same rules as the compile-time overload selection. For this macro to work, -at the very least there has to be an overload with a -@relativeref{Corrade,Cpu::ScalarT} argument. See +parameter, all returning a function pointer of the same type, creates a +function with signature @cpp function(Cpu::Features) @ce which will select +among the overloads using a runtime-specified +@relativeref{Corrade,Cpu::Features}, using the same rules as the compile-time +overload selection. For this macro to work, at the very least there has to be +an overload with a @relativeref{Corrade,Cpu::ScalarT} argument. See @ref Cpu-usage-automatic-runtime-dispatch for more information and an example. This function works with just a single base CPU instruction tag such as @@ -1687,8 +1687,8 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. "the new preprocessor" is enabled it wouldn't be possible to get rid of the CORRADE_CPU_SELECT() macro from there. */ #ifdef CORRADE_TARGET_X86 -#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ - type function(Corrade::Cpu::Features features) { \ +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ if(features & Corrade::Cpu::Avx512f) \ return function(Corrade::Cpu::Avx512f); \ if(features & Corrade::Cpu::Avx2) \ @@ -1708,8 +1708,8 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. return function(Corrade::Cpu::Scalar); \ } #elif defined(CORRADE_TARGET_ARM) -#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ - type function(Corrade::Cpu::Features features) { \ +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ if(features & Corrade::Cpu::NeonFp16) \ return function(Corrade::Cpu::NeonFp16); \ if(features & Corrade::Cpu::NeonFma) \ @@ -1719,15 +1719,15 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. return function(Corrade::Cpu::Scalar); \ } #elif defined(CORRADE_TARGET_WASM) -#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ - type function(Corrade::Cpu::Features features) { \ +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ if(features & Corrade::Cpu::Simd128) \ return function(Corrade::Cpu::Simd128); \ return function(Corrade::Cpu::Scalar); \ } #else -#define CORRADE_CPU_DISPATCHER_BASE(type, function) \ - type function(Corrade::Cpu::Features features) { \ +#define CORRADE_CPU_DISPATCHER_BASE(function) \ + decltype(function(Corrade::Cpu::Scalar)) function(Corrade::Cpu::Features features) { \ return function(Corrade::Cpu::Scalar); \ } #endif @@ -1774,8 +1774,8 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. /* CORRADE_CPU_DISPATCHER() specialization for 0 extra instruction sets. Basically equivalent to CORRADE_CPU_DISPATCHER_BASE() except for the extra CORRADE_CPU_SELECT() macro. */ -#define _CORRADE_CPU_DISPATCHER0(type, function) \ - type function(Corrade::Cpu::Features features) { \ +#define _CORRADE_CPU_DISPATCHER0(function) \ + decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function(Corrade::Cpu::Features features) { \ _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, ) \ } @@ -1785,11 +1785,11 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. reasonable. I attempted adding an "unrolled" _CORRADE_CPU_DISPATCHER2() but it made everything significantly worse on both compilers, more than doubling the amount of generated code. */ -#define _CORRADE_CPU_DISPATCHERn(type, function, ...) \ - template CORRADE_ALWAYS_INLINE type function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags) { \ +#define _CORRADE_CPU_DISPATCHERn(function, ...) \ + template CORRADE_ALWAYS_INLINE decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags) { \ _CORRADE_CPU_DISPATCHER_IMPLEMENTATION(function, |Corrade::Cpu::Implementation::Tags{Corrade::Cpu::Implementation::Init}) \ } \ - template CORRADE_ALWAYS_INLINE type function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags extra, First first, Next... next) { \ + template CORRADE_ALWAYS_INLINE decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function ## Internal(Corrade::Cpu::Features features, Corrade::Cpu::Implementation::Tags extra, First first, Next... next) { \ static_assert(!(static_cast(Corrade::Cpu::Implementation::tags(First{Corrade::Cpu::Implementation::Init})) & Corrade::Cpu::Implementation::BaseTagMask), \ "only extra instruction set tags should be explicitly listed"); \ if(features & first) \ @@ -1797,7 +1797,7 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. else \ return function ## Internal(features, extra, next...); \ } \ - type function(Corrade::Cpu::Features features) { \ + decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function(Corrade::Cpu::Features features) { \ return function ## Internal(features, Corrade::Cpu::Implementation::Tags<0>{Corrade::Cpu::Implementation::Init}, __VA_ARGS__); \ } @@ -1813,14 +1813,14 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. @m_since_latest Given a set of function overloads named @p function that accept a CPU tag -combination wrapped in @ref CORRADE_CPU_DECLARE() as a parameter and return a -function pointer of @p type, creates a function with signature -@cpp type function(Cpu::Features) @ce which will select among the overloads -using a runtime-specified @relativeref{Corrade,Cpu::Features}, using the same -rules as the compile-time overload selection. The extra instruction sets -considered in the overload selection are specified as additional parameters to -the macro, specifying none is valid as well. For this macro to work, at the -very least there has to be an overload with a +combination wrapped in @ref CORRADE_CPU_DECLARE() as a parameter, all returning +a function pointer of the same type, creates a function with signature +@cpp function(Cpu::Features) @ce which will select among the overloads using a +runtime-specified @relativeref{Corrade,Cpu::Features}, using the same rules as +the compile-time overload selection. The extra instruction sets considered in +the overload selection are specified as additional parameters to the macro, +specifying none is valid as well. For this macro to work, at the very least +there has to be an overload with a @ref Corrade::Cpu::Scalar "CORRADE_CPU_DECLARE(Cpu::Scalar)" argument. See @ref Cpu-usage-automatic-runtime-dispatch for more information and an example. @@ -1828,17 +1828,17 @@ For a dispatch using just the base instruction set use @ref CORRADE_CPU_DISPATCHER_BASE() instead. */ #ifdef DOXYGEN_GENERATING_OUTPUT -#define CORRADE_CPU_DISPATCHER(type, function, ...) +#define CORRADE_CPU_DISPATCHER(function, ...) #elif !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) -#define CORRADE_CPU_DISPATCHER(type, ...) \ - _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(type, __VA_ARGS__) +#define CORRADE_CPU_DISPATCHER(...) \ + _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__) #else /* Workaround for MSVC not being able to expand __VA_ARGS__ correctly. Would work with /Zc:preprocessor or /experimental:preprocessor, but I'm not enabling that globally yet. Source: https://stackoverflow.com/a/5134656 */ #define _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS(x) x -#define CORRADE_CPU_DISPATCHER(type, ...) \ - _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS( _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(type, __VA_ARGS__)) +#define CORRADE_CPU_DISPATCHER(...) \ + _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS( _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__)) #endif /** diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index fe11dd5b0..eb42fcfac 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -873,7 +873,7 @@ CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntime(Cpu::Simd128T) { #error #endif -CORRADE_CPU_DISPATCHER_BASE(DispatchRuntimeT, dispatchRuntime) +CORRADE_CPU_DISPATCHER_BASE(dispatchRuntime) #endif void CpuTest::tagDispatchRuntime() { @@ -925,7 +925,7 @@ CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtra(CORRADE_CPU_DECLARE( return []() -> const char* { return "AVX2+FMA"; }; } -CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtra, Cpu::AvxF16c, Cpu::AvxFma) +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtra, Cpu::AvxF16c, Cpu::AvxFma) #endif void CpuTest::tagDispatchRuntimeExtra() { @@ -968,7 +968,7 @@ CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraCombination(CORRADE_C return []() -> const char* { return "AVX2+POPCNT+LZCNT"; }; } -CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraCombination, Cpu::Popcnt|Cpu::Lzcnt) +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraCombination, Cpu::Popcnt|Cpu::Lzcnt) #endif void CpuTest::tagDispatchRuntimeExtraCombination() { @@ -1000,7 +1000,7 @@ CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Avx)) { return []() -> const char* { return "AVX"; }; } -CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraZeroExtra) +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraZeroExtra) #elif defined(CORRADE_TARGET_ARM) CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Neon)) { return []() -> const char* { return "NEON"; }; @@ -1009,13 +1009,13 @@ CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU return []() -> const char* { return "NEON FMA"; }; } -CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraZeroExtra) +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraZeroExtra) #elif defined(CORRADE_TARGET_WASM) CORRADE_ALWAYS_INLINE DispatchRuntimeT dispatchRuntimeExtraZeroExtra(CORRADE_CPU_DECLARE(Cpu::Simd128)) { return []() -> const char* { return "SIMD128"; }; } -CORRADE_CPU_DISPATCHER(DispatchRuntimeT, dispatchRuntimeExtraZeroExtra) +CORRADE_CPU_DISPATCHER(dispatchRuntimeExtraZeroExtra) #else #error #endif @@ -1066,7 +1066,7 @@ CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(C return []() -> Cpu::Features { return Cpu::Avx|Cpu::AvxFma; }; } -CORRADE_CPU_DISPATCHER(DispatchedT, dispatchedImplementation, Cpu::AvxFma) +CORRADE_CPU_DISPATCHER(dispatchedImplementation, Cpu::AvxFma) #elif defined(CORRADE_TARGET_ARM) CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::NeonFp16)) { return []() -> Cpu::Features { return Cpu::NeonFp16; }; @@ -1075,13 +1075,13 @@ CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(C return []() -> Cpu::Features { return Cpu::Neon; }; } -CORRADE_CPU_DISPATCHER(DispatchedT, dispatchedImplementation) +CORRADE_CPU_DISPATCHER(dispatchedImplementation) #elif defined(CORRADE_TARGET_WASM) CORRADE_ALWAYS_INLINE DispatchedT dispatchedImplementation(CORRADE_CPU_DECLARE(Cpu::Simd128)) { return []() -> Cpu::Features { return Cpu::Simd128; }; } -CORRADE_CPU_DISPATCHER(DispatchedT, dispatchedImplementation) +CORRADE_CPU_DISPATCHER(dispatchedImplementation) #endif /* CORRADE_NEVER_INLINE to make it possible to look at the disassembly. The From afbb92e28ae810a2576fcd081f43503283a09ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Jul 2022 15:57:28 +0200 Subject: [PATCH 34/79] doc: refresh minor bits in Developers Guide. I didn't even know this page existed, heh. --- doc/corrade-developers.dox | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/corrade-developers.dox b/doc/corrade-developers.dox index 214cf9a2a..4e1b040e9 100644 --- a/doc/corrade-developers.dox +++ b/doc/corrade-developers.dox @@ -119,7 +119,7 @@ not strictly required to follow them to the point. - make sure they are mentioned in the library documentation - make sure they are mentioned in building and CMake docs - make sure they are mentioned in `CREDITS.md` - - make sure AppVeyor and Travis downloads them (based on platform + - make sure CircleCI and AppVeyor downloads them (based on platform support) 13. Mention the library in `doc/corrade-changelog.dox` 14. Build documentation: @@ -411,19 +411,20 @@ only in inverse. @section corrade-developers-port Checklist for adding or removing a port -1. Add a new `TARGET_*` variable: +1. Add a new `CORRADE_TARGET_*` CMake variable: - to root `CMakeLists.txt`, which either gets enabled automatically based on system introspection or is exposed through a @cmake option() @ce command - to the list of variables extracted out of `configure.h` in `modules/FindCorrade.cmake` -2. Add a `CORRADE_TARGET_*` variable: - - set it in root `CMakeLists.txt` in case `TARGET_*` is enabled - - add it as a @cpp #cmakedefine @ce macro to `src/Corrade/configure.h.cmake` +2. Add a `CORRADE_TARGET_*` preprocessor variable: + - add it as a @cpp #cmakedefine @ce macro to + `src/Corrade/configure.h.cmake` - add documentation for it to `src/Corrade/Corrade.h` - mention it in `modules/FindCorrade.cmake` docs - mention it in `doc/corrade-cmake.dox` and `doc/building-corrade.dox` -3. Add a new Travis / AppVeyor matrix build for this port (or update existing) +3. Add a new CircleCI / AppVeyor matrix build for this port (or update + existing) 4. Add a new `PKGBUILD-*` file in `package/archlinux` for testing (or update existing) 5. Enable or disable functionality using @cmake if(CORRADE_TARGET_*) @ce in From b5f4445b174e7cc70bfba32b1704e481ac8b783b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Jul 2022 18:00:40 +0200 Subject: [PATCH 35/79] Add CORRADE_TARGET_BMI1, _ENABLE_BMI1 and Cpu::Bmi1. Because apparently LZCNT isn't enough, I need also TZCNT. --- doc/corrade-changelog.dox | 15 ++-- doc/cpu.dot | 2 + src/Corrade/Corrade.h | 18 +++++ src/Corrade/Cpu.cpp | 2 + src/Corrade/Cpu.h | 120 ++++++++++++++++++++++++---- src/Corrade/Test/CpuTest.cpp | 24 +++++- src/Corrade/Utility/IntrinsicsAvx.h | 20 ++++- src/Corrade/configure.h.cmake | 20 +++-- 8 files changed, 185 insertions(+), 36 deletions(-) diff --git a/doc/corrade-changelog.dox b/doc/corrade-changelog.dox index d707dfd99..8fdc99b31 100644 --- a/doc/corrade-changelog.dox +++ b/doc/corrade-changelog.dox @@ -46,14 +46,13 @@ namespace Corrade { - New @ref CORRADE_TARGET_SSE3, @ref CORRADE_TARGET_SSSE3, @ref CORRADE_TARGET_SSE41, @ref CORRADE_TARGET_SSE42, @ref CORRADE_TARGET_POPCNT, @ref CORRADE_TARGET_LZCNT, - @ref CORRADE_TARGET_AVX, @ref CORRADE_TARGET_AVX_F16C, - @ref CORRADE_TARGET_AVX_FMA, @ref CORRADE_TARGET_AVX2, - @ref CORRADE_TARGET_AVX512F, @ref CORRADE_TARGET_NEON, - @ref CORRADE_TARGET_NEON_FMA, @ref CORRADE_TARGET_NEON_FP16 and - @ref CORRADE_TARGET_SIMD128 preprocessor variables added to the already - existing - @ref CORRADE_TARGET_SSE2 for detecting enabled instruction sets on x86, ARM - and WebAssembly + @ref CORRADE_TARGET_BMI1, @ref CORRADE_TARGET_AVX, + @ref CORRADE_TARGET_AVX_F16C, @ref CORRADE_TARGET_AVX_FMA, + @ref CORRADE_TARGET_AVX2, @ref CORRADE_TARGET_AVX512F, + @ref CORRADE_TARGET_NEON, @ref CORRADE_TARGET_NEON_FMA, + @ref CORRADE_TARGET_NEON_FP16 and @ref CORRADE_TARGET_SIMD128 preprocessor + variables added to the already existing @ref CORRADE_TARGET_SSE2 for + detecting enabled instruction sets on x86, ARM and WebAssembly - New @ref CORRADE_TARGET_32BIT preprocessor variable for cross-platform detection of 32-bit builds - New @ref Cpu namespace that provides building blocks for compile-time and diff --git a/doc/cpu.dot b/doc/cpu.dot index 215863149..b3f1174a2 100644 --- a/doc/cpu.dot +++ b/doc/cpu.dot @@ -42,10 +42,12 @@ digraph "x86 instruction family tree" { F16C [width=0.75 fontsize=13 class="m-primary"] POPCNT [width=0.75 fontsize=13 margin="0,0" class="m-primary"] LZCNT [width=0.75 fontsize=13 class="m-primary"] + BMI1 [width=0.75 fontsize=13 class="m-primary"] POPCNT -> SSE3 [style=invis] LZCNT -> SSE3 [style=invis] LZCNT -> POPCNT [style=invis] + BMI1 -> LZCNT [style=invis] FMA3 -> AVX [class="m-primary"] F16C -> AVX [class="m-primary"] diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index b35989038..ffcb6bba4 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -643,6 +643,24 @@ instead. #define CORRADE_TARGET_LZCNT #undef CORRADE_TARGET_LZCNT +/** +@brief Target with BMI1 instructions +@m_since_latest + +Defined on @ref CORRADE_TARGET_X86 "x86" if +[BMI1](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#BMI1_(Bit_Manipulation_Instruction_Set_1)) +including the `TZCNT` instruction is enabled at compile time (on GCC/Clang it's +`-mbmi`, MSVC doesn't have a direct option but it's assumed to be implied by +`/arch:AVX`). However note that the `TZCNT` instruction has encoding compatible +with an earlier `BSF` instruction which has a slightly different behavior. To +avoid wrong results if it isn't available, prefer to detect its presence with +@ref Cpu::runtimeFeatures() instead. +@see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Bmi1}, + @ref CORRADE_ENABLE_BMI1 +*/ +#define CORRADE_TARGET_BMI1 +#undef CORRADE_TARGET_BMI1 + /** @brief AVX target @m_since_latest diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index 63311d05a..ca7f2c799 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -245,6 +245,7 @@ Features runtimeFeatures() { #else __cpuid_count(7, 0, cpuid.e.ax, cpuid.e.bx, cpuid.e.cx, cpuid.e.dx); #endif + if(cpuid.e.bx & (1 << 3)) out |= TypeTraits::Index; if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; } @@ -343,6 +344,7 @@ Utility::Debug& operator<<(Utility::Debug& debug, Features value) { even in case of Cpu::Default, where it's just one */ _c(Popcnt) _c(Lzcnt) + _c(Bmi1) _c(AvxF16c) _c(AvxFma) #elif defined(CORRADE_TARGET_ARM) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 7fcee3d84..06671c847 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -127,7 +127,7 @@ overload candidate for the set of CPU instruction features enabled at compile @m_class{m-block m-warning} -@par SSE3, SSSE3, SSE4.1/SSE4.2, POPCNT, LZCNT, AVX F16C and AVX FMA on MSVC +@par SSE3, SSSE3, SSE4.1/SSE4.2, POPCNT, LZCNT, BMI1, AVX F16C and AVX FMA on MSVC A special case worth mentioning are SSE3 and newer instructions on Windows. MSVC only provides a very coarse `/arch:SSE2`, `/arch:AVX` and `/arch:AVX2` for either @ref Sse2, @ref Avx or @ref Avx2, but nothing in between. That @@ -519,6 +519,20 @@ struct LzcntT { #endif }; +/** +@brief BMI1 tag type + +Available only on @ref CORRADE_TARGET_X86 "x86". See the @ref Cpu namespace +and the @ref Bmi1 tag for more information. +@see @ref tag(), @ref features() +*/ +struct Bmi1T { + #ifndef DOXYGEN_GENERATING_OUTPUT + /* Explicit constructor to avoid ambiguous calls when using {} */ + constexpr explicit Bmi1T(Implementation::InitT) {} + #endif +}; + /** @brief AVX tag type @@ -633,12 +647,16 @@ template<> struct TypeTraits { enum: unsigned int { Index = 1 << (1 + Implementation::ExtraTagBitOffset) }; static const char* name() { return "Lzcnt"; } }; -template<> struct TypeTraits { +template<> struct TypeTraits { enum: unsigned int { Index = 1 << (2 + Implementation::ExtraTagBitOffset) }; + static const char* name() { return "Bmi1"; } +}; +template<> struct TypeTraits { + enum: unsigned int { Index = 1 << (3 + Implementation::ExtraTagBitOffset) }; static const char* name() { return "AvxF16c"; } }; template<> struct TypeTraits { - enum: unsigned int { Index = 1 << (3 + Implementation::ExtraTagBitOffset) }; + enum: unsigned int { Index = 1 << (4 + Implementation::ExtraTagBitOffset) }; static const char* name() { return "AvxFma"; } }; #endif @@ -803,7 +821,8 @@ constexpr Sse42T Sse42{Implementation::Init}; instructions. Available only on @ref CORRADE_TARGET_X86 "x86". This instruction set is treated as an *extra*, i.e. is neither a superset of nor implied by any other instruction set. See @ref Cpu-usage-extra for more information. -@see @ref Lzcnt, @ref CORRADE_TARGET_POPCNT, @ref CORRADE_ENABLE_POPCNT +@see @ref Lzcnt, @ref Bmi1, @ref CORRADE_TARGET_POPCNT, + @ref CORRADE_ENABLE_POPCNT */ constexpr PopcntT Popcnt{Implementation::Init}; @@ -819,10 +838,29 @@ Note that this instruction has encoding compatible with an earlier `BSR` instruction which has a slightly different behavior. To avoid wrong results if it isn't available, prefer to always detect its presence with @ref runtimeFeatures() instead of a compile-time check. -@see @ref Popcnt, @ref CORRADE_TARGET_LZCNT, @ref CORRADE_ENABLE_LZCNT +@see @ref Popcnt, @ref Bmi1, @ref CORRADE_TARGET_LZCNT, + @ref CORRADE_ENABLE_LZCNT */ constexpr LzcntT Lzcnt{Implementation::Init}; +/** +@brief BMI1 tag + +[BMI1](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#BMI1_(Bit_Manipulation_Instruction_Set_1)) +instructions, including `TZCNT`. Available only on +@ref CORRADE_TARGET_X86 "x86". This instruction set is treated as an *extra*, +i.e. is neither a superset of nor implied by any other instruction set. See +@ref Cpu-usage-extra for more information. + +Note that the `TZCNT` instruction has encoding compatible with an earlier `BSF` +instruction which has a slightly different behavior. To avoid wrong results if +it isn't available, prefer to always detect its presence with +@ref runtimeFeatures() instead of a compile-time check. +@see @ref Popcnt, @ref Lzcnt, @ref CORRADE_TARGET_BMI1, + @ref CORRADE_ENABLE_BMI1 +*/ +constexpr Bmi1T Bmi1{Implementation::Init}; + /** @brief AVX tag @@ -934,7 +972,7 @@ enum: unsigned int { BaseTagMask = (1 << ExtraTagBitOffset) - 1, ExtraTagMask = 0xffffffffu & ~BaseTagMask, #ifdef CORRADE_TARGET_X86 - ExtraTagCount = 4, + ExtraTagCount = 5, #else ExtraTagCount = 0, #endif @@ -1140,6 +1178,9 @@ typedef Implementation::Tags< #ifdef CORRADE_TARGET_LZCNT TypeTraits::Index| #endif + #ifdef CORRADE_TARGET_BMI1 + TypeTraits::Index| + #endif #ifdef CORRADE_TARGET_AVX_FMA TypeTraits::Index| #endif @@ -1202,6 +1243,7 @@ a combination of these: - @ref Popcnt if @ref CORRADE_TARGET_POPCNT is defined - @ref Lzcnt if @ref CORRADE_TARGET_LZCNT is defined +- @ref Bmi1 if @ref CORRADE_TARGET_BMI1 is defined - @ref AvxFma if @ref CORRADE_TARGET_AVX_FMA is defined - @ref AvxF16c if @ref CORRADE_TARGET_AVX_F16C is defined @@ -1525,9 +1567,9 @@ namespace Implementation { @brief CPU instruction sets enabled at compile time On @ref CORRADE_TARGET_X86 "x86" returns a combination of @ref Sse2, @ref Sse3, -@ref Ssse3, @ref Sse41, @ref Sse42, @ref Popcnt, @ref Lzcnt, @ref Avx, -@ref AvxF16c, @ref AvxFma, @ref Avx2 and @ref Avx512f based on what all -@ref CORRADE_TARGET_SSE2 etc. preprocessor variables are defined. +@ref Ssse3, @ref Sse41, @ref Sse42, @ref Popcnt, @ref Lzcnt, @ref Bmi1, +@ref Avx, @ref AvxF16c, @ref AvxFma, @ref Avx2 and @ref Avx512f based on what +all @ref CORRADE_TARGET_SSE2 etc. preprocessor variables are defined. On @ref CORRADE_TARGET_ARM "ARM", returns a combination of @ref Neon, @ref NeonFma and @ref NeonFp16 based on what all @ref CORRADE_TARGET_NEON etc. @@ -1565,6 +1607,9 @@ constexpr Features compiledFeatures() { #ifdef CORRADE_TARGET_LZCNT TypeTraits::Index| #endif + #ifdef CORRADE_TARGET_BMI1 + TypeTraits::Index| + #endif #ifdef CORRADE_TARGET_AVX TypeTraits::Index| #endif @@ -1603,10 +1648,11 @@ constexpr Features compiledFeatures() { On @ref CORRADE_TARGET_X86 "x86" and GCC, Clang or MSVC uses the [CPUID](https://en.wikipedia.org/wiki/CPUID) builtin to check for the @ref Sse2, @ref Sse3, @ref Ssse3, @ref Sse41, @ref Sse42, @ref Popcnt, -@ref Lzcnt, @ref Avx, @ref AvxF16c, @ref AvxFma, @ref Avx2 and @ref Avx512f -runtime features. @ref Avx needs OS support as well, if it's not present, no -following flags are checked either. On compilers other than GCC, Clang and MSVC -the function is @cpp constexpr @ce and delegates into @ref compiledFeatures(). +@ref Lzcnt, @ref Bmi1, @ref Avx, @ref AvxF16c, @ref AvxFma, @ref Avx2 and +@ref Avx512f runtime features. @ref Avx needs OS support as well, if it's not +present, no following flags including @ref Bmi1 are checked either. On +compilers other than GCC, Clang and MSVC the function is @cpp constexpr @ce and +delegates into @ref compiledFeatures(). On @ref CORRADE_TARGET_ARM "ARM" and Linux or Android API level 18+ uses @m_class{m-doc-external} [getauxval()](https://man.archlinux.org/man/getauxval.3), or on ARM macOS and iOS uses @m_class{m-doc-external} [sysctlbyname()](https://developer.apple.com/documentation/kernel/1387446-sysctlbyname) @@ -2202,7 +2248,7 @@ instructions inside a function annotated with this macro without having to specify `-mlzcnt` for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE variants and POPCNT this macro is not defined on -@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, AVX and newer +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, BMI1, AVX and newer intrinsics are provided only if enabled on compiler command line. Not defined on other compilers or architectures. @@ -2243,6 +2289,52 @@ may need to specify it together with others. See #define CORRADE_ENABLE_LZCNT #endif +/** +@brief Enable BMI1 for given function +@m_since_latest + +On @ref CORRADE_TARGET_X86 "x86" GCC, Clang expands to +@cpp __attribute__((__target__("bmi"))) @ce, allowing use of the +[BMI1](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#BMI1_(Bit_Manipulation_Instruction_Set_1)) +instructions inside a function annotated with this macro without having to +specify `-mbmi` for the whole compilation unit. On x86 MSVC expands to nothing, +as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE +variants and POPCNT this macro is not defined on +@ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, BMI1, AVX and newer +intrinsics are provided only if enabled on compiler command line. Not defined on +other compilers or architectures. + +As a special case, if @ref CORRADE_TARGET_BMI1 is defined (meaning BMI1 is +enabled for the whole compilation unit), this macro is defined as empty on all +compilers. + +Neither a superset nor implied by any other `CORRADE_ENABLE_*` macro, so you +may need to specify it together with others. See +@ref Cpu-usage-target-attributes for more information and usage example. + +@m_class{m-note m-info} + +@par + If you target GCC 4.8, you may also want to use + @ref Corrade/Utility/IntrinsicsAvx.h instead of + @cpp #include @ce to be able to access the intrinsics on this + compiler. + +@see @relativeref{Corrade,Cpu::Bmi1} +*/ +#if defined(CORRADE_TARGET_BMI1) || defined(DOXYGEN_GENERATING_OUTPUT) +#define CORRADE_ENABLE_BMI1 +#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +#define CORRADE_ENABLE_BMI1 __attribute__((__target__("bmi"))) +/* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, + still present in Jul 2022, meaning we can only use these if __BMI__ is + defined. Funnily enough the older headers don't have this on their own, only + . Also I don't think "Actually using intrinsics on Windows + already requires the right /arch: settings" is correct. */ +#elif defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE_BMI1 +#endif + /** @brief Enable AVX for given function @m_since_latest diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index eb42fcfac..8f2d018db 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -171,6 +171,7 @@ CpuTest::CpuTest() { &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, + &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, &CpuTest::enableMacros, @@ -655,11 +656,11 @@ void CpuTest::priority() { /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus zero, times the count of extra tags plus one */ CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Scalar)), 0); - CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Sse2)), 1*5); - CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2)), 7*5); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Sse2)), 1*6); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2)), 7*6); /* Base tag + extra tags is a sum of the two */ - CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), 7*5 + 2); + CORRADE_COMPARE(priorityValue(Cpu::Implementation::priority(Cpu::Avx2|Cpu::AvxFma|Cpu::AvxF16c)), 7*6 + 2); #elif defined(CORRADE_TARGET_ARM) /* Base tag alone is its BitIndex, where the Scalar is the lowest, thus zero, times one as there are no extra tags */ @@ -1317,6 +1318,23 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_LZCNT int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_BMI1 int callInstructionFor() { + /* Just lzcnt alone; using volatile to prevent this from being folded into + a constant */ + volatile int a = 0x6583a000; /* 0x0005c1a6 but bit-reversed */ + /* GCC and Clang have __tzcnt_u32() as well, but MSVC has only a single + underscore. */ + unsigned int count = _tzcnt_u32(a); + + /* Also verify that it does the right thing for 0. If misdetected and the + BSR fallback gets used, this would return something random here. */ + CORRADE_COMPARE(_tzcnt_u32(0), 32); + + CORRADE_COMPARE(count, 13); + return count; +} +#endif #ifdef CORRADE_ENABLE_AVX template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX int callInstructionFor() { __m256d a = _mm256_set_pd(5.47, 2.23, 7.62, 0.5); diff --git a/src/Corrade/Utility/IntrinsicsAvx.h b/src/Corrade/Utility/IntrinsicsAvx.h index 280009cec..236e4c045 100644 --- a/src/Corrade/Utility/IntrinsicsAvx.h +++ b/src/Corrade/Utility/IntrinsicsAvx.h @@ -25,15 +25,15 @@ */ /** @file -@brief Intrinsics for x86 LZCNT, AVX, AVX F16C, AVX FMA and AVX2 instructions +@brief Intrinsics for x86 LZCNT, BMI1, AVX, AVX F16C, AVX FMA and AVX2 instructions @m_since_latest Equivalent to @cpp #include @ce on most compilers except for GCC 4.8, where it contains an additional workaround to make the instructions available with just the @ref CORRADE_ENABLE_AVX, @ref CORRADE_ENABLE_AVX_F16C, @ref CORRADE_ENABLE_AVX_FMA or @ref CORRADE_ENABLE_AVX2 function attributes -instead of having to specify `-mlzcnt`, `-mavx`, `-mf16c`, `-mfma` or `-mavx2` -for the whole compilation unit. +instead of having to specify `-mlzcnt`, `-mbmi`, `-mavx`, `-mf16c`, `-mfma` or +`-mavx2` for the whole compilation unit. As AVX-512 is supported only since GCC 4.9, which doesn't need this workaround, it's not handled here. @@ -58,7 +58,7 @@ it's not handled here. instructions are included below by defining target("avx2") and directly pulling in , and only doing that on GCC 4.8, as all other compilers have everything already included from the top-level . - Same then goes for F16C, FMA and LZCNT, which all also have such weird + Same then goes for F16C, FMA, LZCNT and BMI1, which all also have such weird interactions when pulled in together. I wonder what impact this has on optimization, but I don't care that much as @@ -132,3 +132,15 @@ it's not handled here. #pragma pop_macro("__LZCNT__") #pragma GCC pop_options #endif + +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 +#pragma GCC push_options +#pragma GCC target("bmi") +#pragma push_macro("__BMI__") +#ifndef __BMI__ +#define __BMI__ +#endif +#include +#pragma pop_macro("__BMI__") +#pragma GCC pop_options +#endif diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 0445d9056..31680c6d1 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -235,8 +235,8 @@ #endif #endif -/* POPCNT and LZCNT on GCC, queried wth `gcc -mpopcnt -dM -E - | grep POPCNT`, - and equivalent for LZCNT. */ +/* POPCNT, LZCNT and BMI1 on GCC, queried wth + `gcc -mpopcnt -dM -E - | grep POPCNT`, and equivalent for others. */ #ifdef CORRADE_TARGET_GCC #ifdef __POPCNT__ #define CORRADE_TARGET_POPCNT @@ -244,6 +244,9 @@ #ifdef __LZCNT__ #define CORRADE_TARGET_LZCNT #endif +#ifdef __BMI__ +#define CORRADE_TARGET_BMI1 +#endif /* There doesn't seem to be any equivalent on MSVC, https://github.com/kimwalisch/libpopcnt assumes POPCNT is on x86 MSVC @@ -254,14 +257,17 @@ #elif defined(CORRADE_TARGET_MSVC) && defined(__AVX__) #define CORRADE_TARGET_POPCNT /* As with other AVX+ intrinsics on clang-cl, these are only included if the - corresponding macro is defined as well, so check for __LZCNT__ there. - Otherwise this would mean the CORRADE_ENABLE_LZCNT macro is defined always, - which would incorrectly imply presence of these intrinsics. Chance is that - __LZCNT__ is possibly implied with /arch:AVX2 or some such, but I don't know - for sure, so this is more robust. */ + corresponding macro is defined as well, so check for __LZCNT__ / __BMI__ + there. Otherwise this would mean the CORRADE_ENABLE_{LZCNT,BMI1} macro is + defined always, which would incorrectly imply presence of these intrinsics. + Chance is that __LZCNT__ / __BMI__ is possibly implied with /arch:AVX or + some such, but I don't know for sure, so this is more robust. */ #if !defined(CORRADE_TARGET_CLANG_CL) || defined(__LZCNT__) #define CORRADE_TARGET_LZCNT #endif +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__BMI__) +#define CORRADE_TARGET_BMI1 +#endif #endif /* On GCC, F16C and FMA have its own define, on MSVC FMA is implied by From 62f9aaad78f0b6921b9588d10a56cb4cd55cc9eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Jul 2022 18:01:26 +0200 Subject: [PATCH 36/79] doc: document a checklist for adding a new CPU feature tag. It's not easy to cover everything, heh. --- doc/corrade-developers.dox | 123 +++++++++++++++++++++++++++++++++++++ src/Corrade/Cpu.cpp | 3 + 2 files changed, 126 insertions(+) diff --git a/doc/corrade-developers.dox b/doc/corrade-developers.dox index 4e1b040e9..8d1fbb443 100644 --- a/doc/corrade-developers.dox +++ b/doc/corrade-developers.dox @@ -437,6 +437,129 @@ only in inverse. In order to remove a port, be sure to touch all places mentioned above, only in inverse. +@section corrade-developers-cpu-tag Checklist for adding / removing a CPU instruction set + +1. Add a new `CORRADE_TARGET_FOO` preprocessor variable to `Corrade.h`: + - With relevant documentation pointing ideally to a Wikipedia page + describing the instruction set + - Mentioning the GCC/Clang `-m` option that enables it and the MSVC + equivalent, if any, or the closest `/arch:` option that implies it + - Linking to related other `CORRADE_TARGET_*` variables, or saying it's + a superset of / implied by another + - Mentioning possible caveats (like with `LZCNT` having a dangerous `BSR` + fallback) +2. Add detection into `configure.h.cmake` + - On GCC / Clang it's usually a preprocessor variable in a form of + `__FOO__`, verify with `echo | gcc -dM -E -mfoo | grep FOO` + - If it's a superset of / implied by another, verify that the other + preprocessor variables are set / imply this option as expected + - Cross-check with Clang and if it has a different behavior drop the + implication + - On MSVC just assume it's enabled by the option that implies it, as + there it's harder to verify + - Special-case clang-cl under the MSVC branch, reusing the `__FOO__` + check for it +3. Add a new `FooT` tag type into `Cpu.h` + - Roughly at the place where it historically appeared among other + extensions (e.g. @ref Cpu::PopcntT is after `Sse42T` and before `AvxT` + but new AVX-512 extensions would go at the end) + - If a superset of / implied by another, wire it into the class + hierarchy (like e.g. @ref Cpu::AvxT), otherwise keep it standalone + (like e.g. @ref Cpu::PopcntT) + - Make the minimal documentation only link to the tag instance +4. Add a @cpp TypeTraits @ce entry. Its value is important for proper + overload resolution: + - If it's in a hierarchy, renumber the other indices so it's larger than + everything before it and smaller than everything after. Worst case, if + the hierarchy becomes larger than 16 items, the `ExtraTagBitOffset` may + need to get increased. + - If it's not, `ExtraTagCount` may need to get updated to account for the + new extra tag. You'll get static assertions if the tag value is too + small. +5. Add a new `Foo` tag: + - Position matching the order in which the tag types were defined + - Documentation similar to the `CORRADE_TARGET_FOO` docs, including the + Wikipedia link and potential caveats + - Mentioning it's either a superset of / implied by another (like e.g. + @ref Cpu::Avx) or an "extra" (like e.g. @ref Cpu::Popcnt) + - Linking to relevant other tags and corresponding `CORRADE_TARGET_FOO` + and `CORRADE_ENABLE_FOO` variables + - List it in the right place in the @ref Utility::Debug output operator + in `Cpu.cpp` --- "extra" tags go after the base ones + - If it seems like a reasonable addition, expand the `doc/cpu.dot` graph + with the new tag +6. Wire it into compile-time detection: + - Inside the @ref Cpu::DefaultBaseT / @ref Cpu::DefaultExtraT typedef and + in the @ref Cpu::compiledFeatures() function, at a place that + corresponds to where it was defined among the tags + - Mentioned in @ref Cpu::DefaultBase / @ref Cpu::DefaultExtra and + @ref Cpu::compiledFeatures() docs, again in proper order + - If it doesn't have a direct mapping to a MSVC `/arch:` option, consider + also listing it among others in the yllow warning block in @ref Cpu + namespace docs +7. Wire it into runtime detection: + - For x86 find the right [CPUID](https://en.wikipedia.org/wiki/CPUID) + bit, order the branch among checks for surrounding bits + - For ARM on Linux and Android check for a corresponding + [HWCAP](https://github.com/torvalds/linux/blame/master/arch/arm64/include/uapi/asm/hwcap.h) + bit, in some cases it may need to read the `AT_HWCAP2` entry as well + which has support further limited -- see docs in the code for more + information. + - For ARM on macOS / iOS check the corresponding + [sysctlbyname()](https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics) + value + - For WebAssembly there's no runtime detection yet. Check if anything + happened with the [feature detection proposal](https://github.com/WebAssembly/feature-detection/blob/main/proposals/feature-detection/Overview.md). + - List it in @ref Cpu::runtimeFeatures() docs, if it's depending on AVX + like e.g. @ref Cpu::Bmi1 on x86 and the dependency isn't obvious, + mention that as well +8. Add a new `CORRADE_ENABLE_FOO` macro: + - With a corresponding @cpp __attribute__((__target__("foo"))) @ce, + mention the matching `-m` option in the docs + - If the instruction set works on clang-cl without having to define + anything, then add the attribute also for clang-cl (like e.g. + @ref CORRADE_ENABLE_SSE42), if it doesn't then explicitly exclude + clang-cl such as with @ref CORRADE_ENABLE_LZCNT or + @ref CORRADE_ENABLE_AVX512F. Reflect that in the documentation --- + either saying that it expands to the target attribute also on clang-cl, + or that it isn't defined there ever. + - Verify and mention the special case with `CORRADE_TARGET_FOO` + - If it's a superset of / implied by any other `-m` option (as discovered + when implementing the `CORRADE_TARGET_FOO` detection), mention that in + the docs -- but only if it's consistent for GCC and Clang. +9. Add a test for the `CORRADE_ENABLE_FOO` macro: + - New `callInstructionFor()` variant for given tag type + - Annotated with the macro and also with an @cpp #ifdef @ce for the macro + around + - Ideally using just that instruction alone (such as is the case with + `POPCNT`), if not then try to use an instruction set that's implied by + it so it doesn't need a second `CORRADE_ENABLE_*`. Clearly mark which + expression uses the tested instruction set. It should verify something + nontrivial (so not just a load/store) and return a non-zero value. + - Be sure to build & run on weird / broken compilers such as GCC 4.8 or + clang-cl. In case of GCC 4.8 the `Utility/Intrinsics*.h` headers may + need to get expanded to correctly pull in the intrinsics without + requiring `-mfoo` globally, see their code for more info. +10. Run `CpuTest` to verify everything: + - Compile-time detection (test with a `-march=native` build or + equivalent) + - Runtime detection, ideally on multiple platforms (AMD/Intel for x86, + phone + Apple M1 + CircleCI for ARM...) + - `CORRADE_ENABLE_*` macro (that a corresponding @cpp enableMacros() @ce + test case is run and gives a reasonable result, use CircleCI for + AVX-512 and new ARM extensions if not available locally) + - If the instruction set isn't detected as supported on a target, verify + that calling it crashes (temporarily comment out the `CORRADE_SKIP()` + in `enableMacros()`) +11. List the new `CORRADE_TARGET_FOO`, `CORRADE_ENABLE_FOO` variables and the + `Cpu::Foo` tag in `doc/changelog.dox` +12. Check the output of `CpuTest` CI jobs for sanity: + - Especially differences between Linux, Windows and macOS + - Between x86, ARM and WebAssembly + - Docker CircleCI x86 and ARM jobs tend to have very recent hardware, + OTOH macOS jobs have outdated Intel CPUs without AVX2, the output + should correspond to that + @section corrade-developers-copyright-year Checklist for updating copyright year 1. Verify there are no uncommitted changes in any repos, as that would diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index ca7f2c799..904644327 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -102,6 +102,9 @@ namespace Implementation { with a value fed from outside, which is then used inside such resolvers. For simplicity this variant is available always and the public Cpu::runtimeFeatures() just delegates to it. */ +/** @todo If AT_HWCAP2 or other bits are needed, it's passed to ifunc resolvers + only since glibc 2.30 (and Android API 30+, which is the same as before): + https://github.com/bminor/glibc/commit/2b8a3c86e7606cf1b0a997dad8af2d45ae8989c3 */ Features runtimeFeatures(const unsigned long caps) { unsigned int out = 0; #ifdef CORRADE_TARGET_32BIT From 1b035d034a39e92099ea9b68a467e4a6a377da74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 20 Jul 2022 12:36:11 +0200 Subject: [PATCH 37/79] Cpu: mention an important difference with CORRADE_ENABLE_* and lambdas. --- doc/snippets/Corrade.cpp | 11 +++++++++++ src/Corrade/Cpu.h | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index 36c1c862d..c722113a4 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -102,6 +102,17 @@ TransformT transformImplementation(Cpu::Avx2T) { CORRADE_CPU_DISPATCHER_BASE(transformImplementation) /* [Cpu-usage-automatic-runtime-dispatch-declare] */ +namespace Baz { +TransformT transformImplementation(Cpu::Avx2T); +/* [Cpu-usage-automatic-runtime-dispatch-target-attributes] */ +#ifdef CORRADE_ENABLE_AVX2 +TransformT transformImplementation(Cpu::Avx2T) { + return [](Containers::ArrayView data) CORRADE_ENABLE_AVX2 { DOXYGEN_ELLIPSIS(static_cast(data);) }; +} +#endif +/* [Cpu-usage-automatic-runtime-dispatch-target-attributes] */ +} + using LookupT = int(*)(int); LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)); LookupT lookupImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2)); diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 06671c847..175593a28 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -287,6 +287,19 @@ actual arguments: @snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-call + + +@m_class{m-block m-info } + +@par Instruction enabling macros + An important difference with the @ref CORRADE_ENABLE_SSE2 "CORRADE_ENABLE_*" + macros is that they now have to go directly next to the lambda as GCC + [currently doesn't propagate the attributes](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80439) + from the wrapper function to the nested lambda. The above AVX variant would + look like this with relevant macros added: +@par + @snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-target-attributes + @subsection Cpu-usage-automatic-runtime-dispatch-extra Automatic runtime dispach with extra instruction sets If the variants are tagged with extra instruction sets instead of just the From 39e125539c372202a2c83e9f79d6bc46d0c316b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 20 Jul 2022 16:12:57 +0200 Subject: [PATCH 38/79] CMake: unify the way sanitizers are checked for. This is how it's now done in the root CMakeLists. --- src/Corrade/Containers/Test/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Corrade/Containers/Test/CMakeLists.txt b/src/Corrade/Containers/Test/CMakeLists.txt index 1b9b09d85..2488b669d 100644 --- a/src/Corrade/Containers/Test/CMakeLists.txt +++ b/src/Corrade/Containers/Test/CMakeLists.txt @@ -45,9 +45,9 @@ if(CORRADE_TARGET_EMSCRIPTEN) set_property(TARGET ContainersGrowableArrayTest APPEND_STRING PROPERTY LINK_FLAGS " -s ALLOW_MEMORY_GROWTH=1") endif() corrade_add_test(ContainersGrowableArraySa___FailTest GrowableArraySanitizerFailTest.cpp) -# Not matching with the -f, as MSVC might use either /f or -f. Source: -# https://devblogs.microsoft.com/cppblog/addresssanitizer-asan-for-windows-with-msvc/ -if(CMAKE_CXX_FLAGS MATCHES "fsanitize=address") +# While Clang and GCC use -fsanitize=whatever, MSVC allows also /fsanitize=, +# so catch both. +if(CMAKE_CXX_FLAGS MATCHES "[-/]fsanitize=address") set_tests_properties(ContainersGrowableArraySa___FailTest PROPERTIES PASS_REGULAR_EXPRESSION "AddressSanitizer: container-overflow") endif() From c1c7a582ef98916f9b0a0c3b14ec8f964157dc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 25 Jul 2022 17:57:03 +0200 Subject: [PATCH 39/79] Cpu: one doesn't simply use multiple CORRADE_ENABLE_ macros on GCC. SIGH computers! I hate you. --- doc/corrade-developers.dox | 10 +- doc/snippets/Corrade.cpp | 5 +- src/Corrade/Cpu.h | 321 ++++++++++++++++++++++++++++++++--- src/Corrade/Test/CpuTest.cpp | 124 ++++++++++++-- 4 files changed, 414 insertions(+), 46 deletions(-) diff --git a/doc/corrade-developers.dox b/doc/corrade-developers.dox index 8d1fbb443..2bc0fbea7 100644 --- a/doc/corrade-developers.dox +++ b/doc/corrade-developers.dox @@ -527,10 +527,16 @@ inverse. - If it's a superset of / implied by any other `-m` option (as discovered when implementing the `CORRADE_TARGET_FOO` detection), mention that in the docs -- but only if it's consistent for GCC and Clang. + - Add a `_CORRADE_ENABLE_FOO` variant for the GCC multiple target + attribute workaround -- one empty for when `CORRADE_TARGET_FOO` is + defined and one just the string with a comma after (`"foo",`) 9. Add a test for the `CORRADE_ENABLE_FOO` macro: - New `callInstructionFor()` variant for given tag type - - Annotated with the macro and also with an @cpp #ifdef @ce for the macro - around + - Annotated with the "function variant" of the macro (thus + @cpp CORRADE_ENABLE(FOO) @ce) and also with an @cpp #ifdef @ce for the + macro around. The function variant should be used in order to test both + the `_CORRADE_ENABLE_FOO` macro (on GCC) and `CORRADE_ENABLE_FOO` (on + Clang and elsewhere). - Ideally using just that instruction alone (such as is the case with `POPCNT`), if not then try to use an instruction set that's implied by it so it doesn't need a second `CORRADE_ENABLE_*`. Clearly mark which diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index c722113a4..77e6e1ec5 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -73,8 +73,9 @@ CORRADE_ENABLE_SSE2 int lookup(CORRADE_CPU_DECLARE(Cpu::Sse2), DOXYGEN_ELLIPSIS( #if defined(CORRADE_ENABLE_SSE41) && \ defined(CORRADE_ENABLE_POPCNT) && \ defined(CORRADE_ENABLE_LZCNT) -CORRADE_ENABLE_SSE41 CORRADE_ENABLE_POPCNT CORRADE_ENABLE_LZCNT -int lookup(CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)) { +CORRADE_ENABLE(SSE41,POPCNT,LZCNT) int lookup( + CORRADE_CPU_DECLARE(Cpu::Sse41|Cpu::Popcnt|Cpu::Lzcnt), DOXYGEN_ELLIPSIS(int)) +{ DOXYGEN_ELLIPSIS(return 0;) } #endif diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 175593a28..cdad8c6a1 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -256,6 +256,10 @@ headers you won't use anyway. In comparison, using the @ref CORRADE_TARGET_SSE2 etc. macros would only make the variant available if the whole compilation unit has a corresponding `-m` or `/arch:` option passed to the compiler. +Finally, the @ref CORRADE_ENABLE() function allows multiple instruction sets to +be enabled at the same time in a more concise way and consistently on both GCC +and Clang. + Definitions of the `lookup()` function variants from above would then look like below with the target attributes added. The extra instruction sets get explicitly enabled as well, in contrast a scalar variant would have no @@ -1722,6 +1726,14 @@ best viable overload. */ #define CORRADE_CPU_SELECT(tag) tag, Corrade::Cpu::Implementation::priority(tag) +#ifndef DOXYGEN_GENERATING_OUTPUT +/* Called from CORRADE_CPU_DISPATCHER() and _CORRADE_ENABLE_CONCATENATE() to + pick a macro implementation based on how many arguments were passed. Source: + https://stackoverflow.com/a/11763277 */ +/** @todo move to Utility/Macros.h once it gets useful elsewhere */ +#define _CORRADE_HELPER_PICK(_0, _1, _2, _3, _4, _5, _6, _7, macroName, ...) macroName +#endif + /** @brief Create a function for a runtime dispatch on a base CPU instruction set @m_since_latest @@ -1859,12 +1871,6 @@ sets into account as well use @ref CORRADE_CPU_DISPATCHER() instead. decltype(function(CORRADE_CPU_SELECT(Corrade::Cpu::Scalar))) function(Corrade::Cpu::Features features) { \ return function ## Internal(features, Corrade::Cpu::Implementation::Tags<0>{Corrade::Cpu::Implementation::Init}, __VA_ARGS__); \ } - -/* Called from CORRADE_CPU_DISPATCHER() to pick either _CORRADE_CPU_DISPATCHER0 - if there are no extra arguments after function or _CORRADE_CPU_DISPATCHERn - if there is 1 to 7 extra args (assuming there isn't a need to have more than - 7). Source: https://stackoverflow.com/a/11763277 */ -#define _CORRADE_CPU_DISPATCHER_PICK(_0, _1, _2, _3, _4, _5, _6, _7, macroName, ...) macroName #endif /** @@ -1890,14 +1896,14 @@ For a dispatch using just the base instruction set use #define CORRADE_CPU_DISPATCHER(function, ...) #elif !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_CPU_DISPATCHER(...) \ - _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__) + _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__) #else /* Workaround for MSVC not being able to expand __VA_ARGS__ correctly. Would work with /Zc:preprocessor or /experimental:preprocessor, but I'm not enabling that globally yet. Source: https://stackoverflow.com/a/5134656 */ #define _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS(x) x #define CORRADE_CPU_DISPATCHER(...) \ - _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS( _CORRADE_CPU_DISPATCHER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__)) + _CORRADE_CPU_DISPATCHER_FFS_MSVC_EXPAND_THIS( _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHERn, _CORRADE_CPU_DISPATCHER0, )(__VA_ARGS__)) #endif /** @@ -2044,12 +2050,18 @@ more information and usage example. @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Sse2} +@see @relativeref{Corrade,Cpu::Sse2}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_SSE2) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE2 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE2 +#endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE_SSE2 __attribute__((__target__("sse2"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE2 "sse2", +#endif #elif defined(CORRADE_TARGET_MSVC) #define CORRADE_ENABLE_SSE2 #endif @@ -2081,14 +2093,20 @@ Superset of @ref CORRADE_ENABLE_SSE2, implied by @ref CORRADE_ENABLE_SSSE3. See @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Sse3} +@see @relativeref{Corrade,Cpu::Sse3}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_SSE3) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE3 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE3 +#endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -msse3 option implies -msse2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse3`) */ #define CORRADE_ENABLE_SSE3 __attribute__((__target__("sse3"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE3 "sse3", +#endif #elif defined(CORRADE_TARGET_MSVC) #define CORRADE_ENABLE_SSE3 #endif @@ -2120,14 +2138,20 @@ Superset of @ref CORRADE_ENABLE_SSE3, implied by @ref CORRADE_ENABLE_SSE41. See @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Ssse3} +@see @relativeref{Corrade,Cpu::Ssse3}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_SSSE3) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSSE3 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSSE3 +#endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -mssse3 option implies -msse2 -msse3 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mssse3`) */ #define CORRADE_ENABLE_SSSE3 __attribute__((__target__("ssse3"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSSE3 "ssse3", +#endif #elif defined(CORRADE_TARGET_MSVC) #define CORRADE_ENABLE_SSSE3 #endif @@ -2159,15 +2183,21 @@ See @ref Cpu-usage-target-attributes for more information and usage example. due to both SSE4.1 and 4.2 intrinsics living in the same header. You can only use @ref CORRADE_ENABLE_SSE42 in this case. -@see @relativeref{Corrade,Cpu::Sse41} +@see @relativeref{Corrade,Cpu::Sse41}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_SSE41) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE41 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE41 +#endif #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 409) || defined(CORRADE_TARGET_CLANG) /* also matches clang-cl */ /* The -msse4.1 option implies -msse2 -msse3 -mssse3 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse4.1`) */ #define CORRADE_ENABLE_SSE41 __attribute__((__target__("sse4.1"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE41 "sse4.1", +#endif #elif defined(CORRADE_TARGET_MSVC) #define CORRADE_ENABLE_SSE41 #endif @@ -2199,15 +2229,21 @@ Superset of @ref CORRADE_ENABLE_SSE41, implied by @ref CORRADE_ENABLE_AVX. See @cpp #include @ce and @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Sse42} +@see @relativeref{Corrade,Cpu::Sse42}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_SSE42) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE42 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE42 +#endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -msse4.2 option implies -msse2 -msse3 -mssse3 -msse4.1 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse4.2`) */ #define CORRADE_ENABLE_SSE42 __attribute__((__target__("sse4.2"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_SSE42 "sse4.2", +#endif #elif defined(CORRADE_TARGET_MSVC) #define CORRADE_ENABLE_SSE42 #endif @@ -2240,12 +2276,18 @@ may need to specify it together with others. See @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Popcnt} +@see @relativeref{Corrade,Cpu::Popcnt}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_POPCNT) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_POPCNT +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_POPCNT +#endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE_POPCNT __attribute__((__target__("popcnt"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_POPCNT "popcnt", +#endif #elif defined(CORRADE_TARGET_MSVC) #define CORRADE_ENABLE_POPCNT #endif @@ -2281,17 +2323,26 @@ may need to specify it together with others. See @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Lzcnt} +@see @relativeref{Corrade,Cpu::Lzcnt}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_LZCNT) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_LZCNT +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_LZCNT +#endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ /* GCC 4.8 needs also -mabm. This option is not needed since 4.9 and is also unrecognized on Clang. */ #if defined(CORRADE_TARGET_CLANG) || __GNUC__*100 + __GNUC_MINOR__ >= 409 #define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_LZCNT "lzcnt", +#endif #else #define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt,abm"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_LZCNT "lzcnt,abm", +#endif #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __LZCNT__ is @@ -2333,12 +2384,18 @@ may need to specify it together with others. See @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Bmi1} +@see @relativeref{Corrade,Cpu::Bmi1}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_BMI1) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_BMI1 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_BMI1 +#endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ #define CORRADE_ENABLE_BMI1 __attribute__((__target__("bmi"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_BMI1 "bmi", +#endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __BMI__ is defined. Funnily enough the older headers don't have this on their own, only @@ -2377,15 +2434,21 @@ Superset of @ref CORRADE_ENABLE_SSE42, implied by @ref CORRADE_ENABLE_AVX2. See @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Avx} +@see @relativeref{Corrade,Cpu::Avx}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_AVX) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX +#endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ /* The -mavx option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx`) */ #define CORRADE_ENABLE_AVX __attribute__((__target__("avx"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX "avx", +#endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __AVX__ is defined. Funnily enough the older headers don't have this on their own, only @@ -2426,14 +2489,20 @@ information and usage example. @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::AvxF16c} +@see @relativeref{Corrade,Cpu::AvxF16c}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_AVX_F16C) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX_F16C +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX_F16C +#endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ /* The -mf16c option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ #define CORRADE_ENABLE_AVX_F16C __attribute__((__target__("f16c"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX_F16C "f16c", +#endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __F16C__ is defined. Funnily enough the older headers don't have this on their own, only @@ -2474,14 +2543,20 @@ information and usage example. @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::AvxFma} +@see @relativeref{Corrade,Cpu::AvxFma}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_AVX_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX_FMA +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX_FMA +#endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ /* The -mfma option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ #define CORRADE_ENABLE_AVX_FMA __attribute__((__target__("fma"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX_FMA "fma", +#endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __FMA__ is defined. Funnily enough the older headers don't have this on their own, only @@ -2521,15 +2596,21 @@ See @ref Cpu-usage-target-attributes for more information and usage example. @cpp #include @ce to be able to access the intrinsics on this compiler. -@see @relativeref{Corrade,Cpu::Avx2} +@see @relativeref{Corrade,Cpu::Avx2}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_AVX2) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX2 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX2 +#endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ /* The -mavx2 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx2`) */ #define CORRADE_ENABLE_AVX2 __attribute__((__target__("avx2"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX2 "avx2", +#endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __AVX2__ is defined. Funnily enough the older headers don't have this on their own, only @@ -2561,15 +2642,21 @@ empty on all compilers. Superset of @ref CORRADE_ENABLE_AVX2. See @ref Cpu-usage-target-attributes for more information and usage example. -@see @relativeref{Corrade,Cpu::Avx512f} +@see @relativeref{Corrade,Cpu::Avx512f}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_AVX512F) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX512F +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX512F +#endif #elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ /* The -mavx512 option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx -mavx2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx512f`) */ #define CORRADE_ENABLE_AVX512F __attribute__((__target__("avx512f"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_AVX512F "avx512f", +#endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __AVX512F__ is defined. Funnily enough the older headers don't have this on their own, only @@ -2602,10 +2689,13 @@ compilers. This is also the case for ARM64, where NEON support is implicit Implied by @ref CORRADE_ENABLE_NEON_FMA. See @ref Cpu-usage-target-attributes for more information and usage example. -@see @relativeref{Corrade,Cpu::Neon} +@see @relativeref{Corrade,Cpu::Neon}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_NEON) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_NEON +#endif /* https://github.com/android/ndk/issues/1066 is the only reported (and ignored) issue I found, feels strange that people would just not use ifunc or target attributes on Android at all and instead put everything in @@ -2613,6 +2703,9 @@ for more information and usage example. ditched GCC, where this works properly. */ #elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) #define CORRADE_ENABLE_NEON __attribute__((__target__("fpu=neon"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_NEON "fpu=neon", +#endif #elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON #endif @@ -2638,13 +2731,19 @@ all compilers. This is also the case for ARM64, where NEON support is implicit Superset of @ref CORRADE_ENABLE_NEON, implied by @ref CORRADE_ENABLE_NEON_FP16. See @ref Cpu-usage-target-attributes for more information and usage example. -@see @relativeref{Corrade,Cpu::NeonFma} +@see @relativeref{Corrade,Cpu::NeonFma}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_NEON_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON_FMA +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_NEON_FMA +#endif /* See CORRADE_ENABLE_NEON above for details about Clang */ #elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) #define CORRADE_ENABLE_NEON_FMA __attribute__((__target__("fpu=neon-vfpv4"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_NEON_FMA "fpu=neon-vfpv4", +#endif #elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON_FMA #endif @@ -2669,13 +2768,19 @@ on all compilers. Superset of @ref CORRADE_ENABLE_NEON_FMA. See @ref Cpu-usage-target-attributes for more information and usage example. -@see @relativeref{Corrade,Cpu::NeonFp16} +@see @relativeref{Corrade,Cpu::NeonFp16}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_NEON_FP16) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON_FP16 +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_NEON_FP16 +#endif /* See CORRADE_ENABLE_NEON above for details about Clang */ #elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) #define CORRADE_ENABLE_NEON_FP16 __attribute__((__target__("arch=armv8.2-a+fp16"))) +#ifdef CORRADE_TARGET_GCC +#define _CORRADE_ENABLE_NEON_FP16 "arch=armv8.2-a+fp16", +#endif #elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON_FP16 #endif @@ -2700,13 +2805,179 @@ is implemented, but likely only for instruction sets building on top of this one. See @ref Cpu-usage-target-attributes for more information and usage example. -@see @relativeref{Corrade,Cpu::Simd128} +@see @relativeref{Corrade,Cpu::Simd128}, @ref CORRADE_ENABLE() */ #if defined(CORRADE_TARGET_SIMD128) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SIMD128 +#ifdef CORRADE_TARGET_GCC /* Future-proofing, GCC has no WASM support ATM */ +#define _CORRADE_ENABLE_SIMD128 +#endif +#endif +#endif + +#ifndef DOXYGEN_GENERATING_OUTPUT +/* GCC treats __attribute__((target("foo"))) __attribute__((target("bar"))) as + if only "bar" was specified, thus it's not possible to just put several + CORRADE_ENABLE_ macros after each other. Instead, the only accepted form is + __attribute__((target("foo,bar"))). Fortunately, string literal + concatenation works here, thus with some extra macro trickery we can produce + __attribute__((target("foo" "," "bar"))). The pieces are _CORRADE_ENABLE_* + variants defined if and only if a corresponding CORRADE_ENABLE_* macro is + defined. These can however be empty, thus it's not possible to just join + them all with "," in between. Instead, the macros themselves have a trailing + comma after the string literal (thus "foo",), which causes the empty macros + be filtered out when passed one after another (without commas) from + _CORRADE_ENABLEn to _CORRADE_ENABLE_CONCATENATE(). */ +#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#define _CORRADE_ENABLE_CONCATENATE0(unused) +#define _CORRADE_ENABLE_CONCATENATE1(v0, unused) \ + __attribute__((__target__(v0))) +#define _CORRADE_ENABLE_CONCATENATE2(v0, v1, unused) \ + __attribute__((__target__(v0 "," v1))) +#define _CORRADE_ENABLE_CONCATENATE3(v0, v1, v2, unused) \ + __attribute__((__target__(v0 "," v1 "," v2))) +#define _CORRADE_ENABLE_CONCATENATE4(v0, v1, v2, v3, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3))) +#define _CORRADE_ENABLE_CONCATENATE5(v0, v1, v2, v3, v4, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3 "," v4))) +#define _CORRADE_ENABLE_CONCATENATE6(v0, v1, v2, v3, v4, v5, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3 "," v4 "," v5))) +#define _CORRADE_ENABLE_CONCATENATE7(v0, v1, v2, v3, v4, v5, v6, unused) \ + __attribute__((__target__(v0 "," v1 "," v2 "," v3 "," v4 "," v5 "," v6))) +#define _CORRADE_ENABLE_CONCATENATE(...) \ + _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_ENABLE_CONCATENATE7, _CORRADE_ENABLE_CONCATENATE6, _CORRADE_ENABLE_CONCATENATE5, _CORRADE_ENABLE_CONCATENATE4, _CORRADE_ENABLE_CONCATENATE3, _CORRADE_ENABLE_CONCATENATE2, _CORRADE_ENABLE_CONCATENATE1, _CORRADE_ENABLE_CONCATENATE0, )(__VA_ARGS__) +/* No _CORRADE_HELPER_PASTE() needed here, as there's enough other indirections + to make that work */ +#define _CORRADE_ENABLE1(v0) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + ) +#define _CORRADE_ENABLE2(v0, v1) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + ) +#define _CORRADE_ENABLE3(v0, v1, v2) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + ) +#define _CORRADE_ENABLE4(v0, v1, v2, v3) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + ) +#define _CORRADE_ENABLE5(v0, v1, v2, v3, v4) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + _CORRADE_ENABLE_ ## v4 \ + ) +#define _CORRADE_ENABLE6(v0, v1, v2, v3, v4, v5) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + _CORRADE_ENABLE_ ## v4 \ + _CORRADE_ENABLE_ ## v5 \ + ) +#define _CORRADE_ENABLE7(v0, v1, v2, v3, v4, v5, v6) \ + _CORRADE_ENABLE_CONCATENATE( \ + _CORRADE_ENABLE_ ## v0 \ + _CORRADE_ENABLE_ ## v1 \ + _CORRADE_ENABLE_ ## v2 \ + _CORRADE_ENABLE_ ## v3 \ + _CORRADE_ENABLE_ ## v4 \ + _CORRADE_ENABLE_ ## v5 \ + _CORRADE_ENABLE_ ## v6 \ + ) +/* None of this is needed for Clang, fortunately, so here the whole thing + expands to just CORRADE_ENABLE_FOO CORRADE_ENABLE_BAR. I hope GCC eventually + fixes this as well, so keeping both variants so I can drop the GCC-specific + one in the future. As another future-proof this also gets used for any + compilers other than MSVC. MSVC's preprocessor won't be able to perform the + delayed expansion so CORRADE_ENABLE() */ +#elif defined(CORRADE_TARGET_CLANG) || !defined(CORRADE_TARGET_MSVC) +/* Using _CORRADE_HELPER_PASTE2() instead of _CORRADE_HELPER_PASTE() here, as + that's enough to make that work and it's less work for the preprocessor. + Concatenating directly doesn't work, unlike in the above case for GCC. */ +#define _CORRADE_ENABLE1(v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) +#define _CORRADE_ENABLE2(v0, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) +#define _CORRADE_ENABLE3(v0, v1, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) +#define _CORRADE_ENABLE4(v0, v1, v2, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) +#define _CORRADE_ENABLE5(v0, v1, v2, v3, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) +#define _CORRADE_ENABLE6(v0, v1, v2, v3, v4, v5) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v5) +#define _CORRADE_ENABLE7(v0, v1, v2, v3, v4, v5, v6) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v5) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v6) +#define _CORRADE_ENABLE8(v0, v1, v2, v3, v4, v5, v6, v7) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v0) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v1) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v2) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v3) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v4) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v5) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v6) \ + _CORRADE_HELPER_PASTE2(CORRADE_ENABLE_, v7) #endif #endif +/** +@brief Enable multiple targets for given function +@m_since_latest + +Accepts a comma-separated list of `CORRADE_ENABLE_*` macro suffixes, +effectively enabling given combination. For the macro to work, all +`CORRADE_ENABLE_*` macros corresponding to the arguments have to be defined, +the common usage pattern is thus in combination with an @cpp #ifdef @ce. See +@ref Cpu-usage-target-attributes for more information and an example. + +When multiple `CORRADE_ENABLE_*` macros are specified one after another, GCC +would pick only the last specified, ignoring the others. For it, the macro +expands into a single combined @cpp __attribute__((__target__(...))) @ce +attribute. For other compilers except MSVC it's just a shorthand for multiple +`CORRADE_ENABLE_*` macros one after another. On MSVC expands to nothing --- +there the functions aren't annotated in anyway and moreover the default +preprocessor behavior would make this extremely tricky to implement. +*/ +#if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) +#define CORRADE_ENABLE(...) _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_ENABLE8, _CORRADE_ENABLE7, _CORRADE_ENABLE6, _CORRADE_ENABLE5, _CORRADE_ENABLE4, _CORRADE_ENABLE3, _CORRADE_ENABLE2, _CORRADE_ENABLE1, )(__VA_ARGS__) +#else +#define CORRADE_ENABLE(...) +#endif + } } diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index 8f2d018db..d2fb5ba8c 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -107,6 +107,9 @@ struct CpuTest: TestSuite::Tester { template void enableMacros(); + void enableMacrosMultiple(); + void enableMacrosMultipleAllEmpty(); + void debug(); void debugPacked(); }; @@ -185,6 +188,9 @@ CpuTest::CpuTest() { &CpuTest::enableMacros, #endif + &CpuTest::enableMacrosMultiple, + &CpuTest::enableMacrosMultipleAllEmpty, + &CpuTest::debug, &CpuTest::debugPacked}); } @@ -1212,7 +1218,7 @@ template int callInstructionFor() { into what code they get actually compiled. Except for the catch-all variant, which isn't interesting for disassembly. */ #ifdef CORRADE_ENABLE_SSE2 -template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE2 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE2) int callInstructionFor() { __m128i a = _mm_set_epi32(0x80808080, 0, 0x80808080, 0); /* All instructions SSE2 */ @@ -1223,7 +1229,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE2 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE3 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE3) int callInstructionFor() { const std::uint32_t a[]{0, 10, 20, 30, 40}; /* SSE3 */ @@ -1241,7 +1247,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE3 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_SSSE3 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSSE3) int callInstructionFor() { __m128i a = _mm_set_epi32(-10, 20, -30, 40); /* SSSE3 */ @@ -1259,7 +1265,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSSE3 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE41 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE41) int callInstructionFor() { __m128 a = _mm_set_ps(5.47f, 2.23f, 7.62f, 0.5f); /* SSE4.1 */ @@ -1277,7 +1283,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE41 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE42 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SSE42) int callInstructionFor() { __m128i a = _mm_set_epi64x(50, 60); __m128i b = _mm_set_epi64x(60, 50); @@ -1294,7 +1300,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SSE42 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_POPCNT int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(POPCNT) int callInstructionFor() { /* Just pocnt alone; using volatile to prevent this from being folded into a constant */ volatile unsigned int a = 0x0005c1a6; @@ -1304,7 +1310,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_POPCNT int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_LZCNT int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(LZCNT) int callInstructionFor() { /* Just lzcnt alone; using volatile to prevent this from being folded into a constant */ volatile int a = 0x0005c1a6; @@ -1319,7 +1325,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_LZCNT int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_BMI1 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(BMI1) int callInstructionFor() { /* Just lzcnt alone; using volatile to prevent this from being folded into a constant */ volatile int a = 0x6583a000; /* 0x0005c1a6 but bit-reversed */ @@ -1336,7 +1342,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_BMI1 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX) int callInstructionFor() { __m256d a = _mm256_set_pd(5.47, 2.23, 7.62, 0.5); /* All instructions AVX */ @@ -1355,7 +1361,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX_F16C int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX_F16C) int callInstructionFor() { /* Values from Magnum::Math::Test::HalfTest::pack() */ __m128 a = _mm_set_ps(0.0f, 123.75f, -0.000351512f, 3.0f); @@ -1374,7 +1380,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX_F16C int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX_FMA int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX_FMA) int callInstructionFor() { /* Values from Magnum::Math::Test::FunctionsTest::fma() */ __m128 a = _mm_set_ps(0.0f, 2.0f, 1.5f, 0.5f); __m128 b = _mm_set_ps(0.0f, 3.0f, 2.0f, -1.0f); @@ -1395,7 +1401,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX_FMA int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX2 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX2) int callInstructionFor() { __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); /* Like callInstructionFor(), but expanded to AVX2 */ @@ -1406,7 +1412,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX2 int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX512F int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(AVX512F) int callInstructionFor() { __m128 a = _mm_set1_ps(5.47f); /* AVX512 */ @@ -1417,7 +1423,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_AVX512F int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(NEON) int callInstructionFor() { int32x4_t a{-10, 20, -30, 40}; /* All instructions NEON */ @@ -1435,7 +1441,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON_FMA int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(NEON_FMA) int callInstructionFor() { /* Values from Magnum::Math::Test::FunctionsTest::fma() */ float32x4_t a{0.0f, 2.0f, 1.5f, 0.5f}; float32x4_t b{0.0f, 3.0f, 2.0f, -1.0f}; @@ -1456,7 +1462,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON_FMA int callInstructionFor CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON_FP16 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(NEON_FP16) int callInstructionFor() { float32x4_t a{5.47f, 2.23f, 7.62f, 0.5f}; float16x4_t b = vcvt_f16_f32(a); @@ -1476,7 +1482,7 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_NEON_FP16 int callInstructionFor< } #endif #ifdef CORRADE_ENABLE_SIMD128 -template<> CORRADE_NEVER_INLINE CORRADE_ENABLE_SIMD128 int callInstructionFor() { +template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SIMD128) int callInstructionFor() { v128_t a = wasm_i32x4_make(-10, 20, -30, 40); /* All instructions SIMD128 */ @@ -1504,6 +1510,90 @@ template void CpuTest::enableMacros() { CORRADE_VERIFY(callInstructionFor()); } +#if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_AVX) +/* If set to 1, should fail on GCC (unless CORRADE_TARGET_AVX is set) */ +#if 0 +CORRADE_ENABLE_AVX2 CORRADE_ENABLE_AVX +#else +/* If CORRADE_TARGET_SSE2 is set (on 64bit), it'll result in just "avx2,avx" */ +CORRADE_ENABLE(AVX2,SSE2,AVX) +#endif +int callInstructionMultiple() { + if(!(Cpu::runtimeFeatures() & Cpu::Avx2)) + CORRADE_SKIP("AVX2 feature not supported"); + + /* Same as callInstructionFor() */ + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* If the AVX2 instructions aren't enabled, this will fail to link */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00); + return mask; +} +#elif defined(CORRADE_ENABLE_NEON_FMA) && defined(CORRADE_ENABLE_NEON) +/* If set to 1, should fail on GCC (unless CORRADE_TARGET_NEON is set) */ +#if 0 +CORRADE_ENABLE_NEON_FMA CORRADE_ENABLE_NEON +#else +CORRADE_ENABLE(NEON_FMA,NEON) +#endif +int callInstructionMultiple() { + if(!(Cpu::runtimeFeatures() & Cpu::NeonFma)) + CORRADE_SKIP("NEON FMA feature not supported"); + + /* Same as callInstructionFor() */ + float32x4_t a{0.0f, 2.0f, 1.5f, 0.5f}; + float32x4_t b{0.0f, 3.0f, 2.0f, -1.0f}; + float32x4_t c{0.0f, 0.75f, 0.25f, 0.1f}; + + union { + float32x4_t v; + float s[4]; + } d; + /* If the FMA instructions aren't enabled, this will fail to link */ + d.v = vfmaq_f32(c, b, a); + + CORRADE_COMPARE(d.s[0], 0.0f); + CORRADE_COMPARE(d.s[1], 6.75f); + CORRADE_COMPARE(d.s[2], 3.25f); + CORRADE_COMPARE(d.s[3], -0.4f); + return d.s[2]; +} +#else +int callInstructionMultiple() { + CORRADE_SKIP("Not enough CORRADE_ENABLE_ macros defined"); +} +#endif + +void CpuTest::enableMacrosMultiple() { + CORRADE_VERIFY(callInstructionMultiple()); +} + +/* If the ENABLE_ macro is empty, it should not result in any __attribute__ + annotation */ +#ifdef CORRADE_TARGET_SSE2 +CORRADE_ENABLE(SSE2) int callInstructionMultipleAllEmpty() { + return 1; +} +#elif defined(CORRADE_TARGET_NEON) +CORRADE_ENABLE(NEON) int callInstructionMultipleAllEmpty() { + return 1; +} +#elif defined(CORRADE_TARGET_SIMD128) +CORRADE_ENABLE(SIMD128) int callInstructionMultipleAllEmpty() { + return 1; +} +#else +int callInstructionMultipleAllEmpty() { + CORRADE_SKIP("No suitable CORRADE_TARGET_ macro defined"); +} +#endif + +void CpuTest::enableMacrosMultipleAllEmpty() { + CORRADE_COMPARE(callInstructionMultipleAllEmpty(), 1); +} + void CpuTest::debug() { /* If more flags get added, this might need to become even more zeros */ const unsigned int dead = 0xde00ad00; From 4c41cb48d884fd822b9d01d108e2d1866bfbafcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 21 Jul 2022 15:53:37 +0200 Subject: [PATCH 40/79] Improve compile-time POPCNT, LZCNT and BMI1 detection on MSVC. Thanks to https://twitter.com/pervognsen/status/1549845044139413504 I managed to find the corresponding piece of code that maps between MSVC's /arch: option and GCC/Clang -march= options, which means I can clearly see that LZCNT and BMI1 isn't implied by /arch:AVX but rather /arch:AVX2. None of this voodoo would be needed if they properly documented the options in the first place instead of handwaving it away with "AVX hardware". FFS. --- src/Corrade/Corrade.h | 12 +++++------ src/Corrade/configure.h.cmake | 39 +++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index ffcb6bba4..893788ef0 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -632,8 +632,8 @@ presence with @ref Cpu::runtimeFeatures(). Defined on @ref CORRADE_TARGET_X86 "x86" if [LZCNT](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation)) is enabled at compile time (on GCC/Clang it's `-mlznct`, MSVC doesn't have a -direct option but it's assumed to be implied by `/arch:AVX`). However note that -this instruction has encoding compatible with an earlier `BSR` instruction +direct option but it's assumed to be implied by `/arch:AVX2`). However note +that this instruction has encoding compatible with an earlier `BSR` instruction which has a slightly different behavior. To avoid wrong results if it isn't available, prefer to detect its presence with @ref Cpu::runtimeFeatures() instead. @@ -651,10 +651,10 @@ Defined on @ref CORRADE_TARGET_X86 "x86" if [BMI1](https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#BMI1_(Bit_Manipulation_Instruction_Set_1)) including the `TZCNT` instruction is enabled at compile time (on GCC/Clang it's `-mbmi`, MSVC doesn't have a direct option but it's assumed to be implied by -`/arch:AVX`). However note that the `TZCNT` instruction has encoding compatible -with an earlier `BSF` instruction which has a slightly different behavior. To -avoid wrong results if it isn't available, prefer to detect its presence with -@ref Cpu::runtimeFeatures() instead. +`/arch:AVX2`). However note that the `TZCNT` instruction has encoding +compatible with an earlier `BSF` instruction which has a slightly different +behavior. To avoid wrong results if it isn't available, prefer to detect its +presence with @ref Cpu::runtimeFeatures() instead. @see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Bmi1}, @ref CORRADE_ENABLE_BMI1 */ diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index 31680c6d1..b33d4c728 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -252,16 +252,23 @@ https://github.com/kimwalisch/libpopcnt assumes POPCNT is on x86 MSVC always, and LZCNT has encoding compatible with BSR so if not available it'll not crash but produce wrong results, sometimes. Enabling them always feels a - bit too much, so restricting that only to AVX+, since one is in SSE4 - intrinsic headers and the other in AVX headers. */ -#elif defined(CORRADE_TARGET_MSVC) && defined(__AVX__) + bit too much, so instead going with what clang-cl uses. There /arch:AVX + matches -march=sandybridge: + https://github.com/llvm/llvm-project/blob/6542cb55a3eb115b1c3592514590a19987ffc498/clang/lib/Driver/ToolChains/Arch/X86.cpp#L46-L58 + and `echo | clang -march=sandybridge -dM -E -` lists only POPCNT, while + /arch:AVX2 matches -march=haswell, which lists also LZCNT, BMI and BMI2. */ +#elif defined(CORRADE_TARGET_MSVC) +/* For extra robustness on clang-cl check the macros explicitly -- as with + other AVX+ intrinsics, these are only included if the corresponding macro is + defined as well. Failing to do so would mean the + CORRADE_ENABLE_AVX_{POPCNT,LZCNT,BMI1} macros are defined always, + incorrectly implying presence of these intrinsics. */ +#ifdef __AVX__ +#if !defined(CORRADE_TARGET_CLANG_CL) || defined(__POPCNT__) #define CORRADE_TARGET_POPCNT -/* As with other AVX+ intrinsics on clang-cl, these are only included if the - corresponding macro is defined as well, so check for __LZCNT__ / __BMI__ - there. Otherwise this would mean the CORRADE_ENABLE_{LZCNT,BMI1} macro is - defined always, which would incorrectly imply presence of these intrinsics. - Chance is that __LZCNT__ / __BMI__ is possibly implied with /arch:AVX or - some such, but I don't know for sure, so this is more robust. */ +#endif +#endif +#ifdef __AVX2__ #if !defined(CORRADE_TARGET_CLANG_CL) || defined(__LZCNT__) #define CORRADE_TARGET_LZCNT #endif @@ -269,6 +276,7 @@ #define CORRADE_TARGET_BMI1 #endif #endif +#endif /* On GCC, F16C and FMA have its own define, on MSVC FMA is implied by /arch:AVX2 (https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86), @@ -285,12 +293,13 @@ #define CORRADE_TARGET_AVX_FMA #endif #elif defined(CORRADE_TARGET_MSVC) && defined(__AVX2__) -/* As with other AVX+ intrinsics on clang-cl, these are only included if the - corresponding macro is defined as well, so check for __F16C__ and __FMA__ - there. Otherwise this would mean the CORRADE_ENABLE_AVX_{16C,FMA} macros are - defined always, which would incorrectly imply presence of these intrinsics. - Chance is that __F16C__ / __FMA__ is possibly implied with /arch:AVX2, but I - don't know for sure, so this is more robust. */ +/* On clang-cl /arch:AVX2 matches -march=haswell: + https://github.com/llvm/llvm-project/blob/6542cb55a3eb115b1c3592514590a19987ffc498/clang/lib/Driver/ToolChains/Arch/X86.cpp#L46-L58 + And `echo | clang -march=haswell -dM -E -` lists both F16C and FMA. However, + for robustness, check the macros explicitly -- as with other AVX+ intrinsics + on clang-cl, these are only included if the corresponding macro is defined + as well. Failing to do so would mean the CORRADE_ENABLE_AVX_{16C,FMA} macros + are defined always, incorrectly implying presence of these intrinsics. */ #if !defined(CORRADE_TARGET_CLANG_CL) || defined(__F16C__) #define CORRADE_TARGET_AVX_F16C #endif From fc82fb4edaea0e668d597389877ddd1f109b09f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 25 Jul 2022 19:58:31 +0200 Subject: [PATCH 41/79] Cpu: don't use assertions in CPUID retrieval code. Those are rather high-level, so if they would fail, they'd make the program crash if called from inside an IFUNC handler. So better just not do anything. --- src/Corrade/Cpu.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index 904644327..cf2e3bdb1 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -28,7 +28,6 @@ #include "Corrade/Containers/StringView.h" #include "Corrade/Utility/Debug.h" -#include "Corrade/Utility/DebugAssert.h" #ifdef CORRADE_TARGET_X86 #include @@ -176,7 +175,7 @@ Features runtimeFeatures() { #ifdef CORRADE_TARGET_MSVC __cpuid(cpuid.data, 1); #elif defined(CORRADE_TARGET_GCC) - CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid(1, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); + __get_cpuid(1, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx); #endif unsigned int out = 0; @@ -230,7 +229,7 @@ Features runtimeFeatures() { Furthermore, Clang 5.0 is 9.3 on Apple, 9.2 has only Clang 4: https://en.wikipedia.org/wiki/Xcode#Toolchain_versions */ #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 701) || (defined(CORRADE_TARGET_CLANG) && ((!defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__ >= 5) || (defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__*100 + __clang_minor__ >= 903))) - CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid_count(7, 0, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); + __get_cpuid_count(7, 0, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx); /* Looking at the above commits, on older GCC, __get_cpuid(7) seems to do what __get_cpuid_count(7, 0) does on new GCC, however that's only since 6.3: @@ -258,7 +257,7 @@ Features runtimeFeatures() { #ifdef CORRADE_TARGET_MSVC __cpuid(cpuid.data, 0x80000001); #elif defined(CORRADE_TARGET_GCC) - CORRADE_INTERNAL_DEBUG_ASSERT_OUTPUT(__get_cpuid(0x80000001, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx)); + __get_cpuid(0x80000001, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx); #endif if(cpuid.e.cx & (1 << 5)) out |= TypeTraits::Index; From 80a587a3fb6ffa84d4110dd02154224af8c91964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 25 Jul 2022 21:01:33 +0200 Subject: [PATCH 42/79] Cpu: inline the CPUID query in the header and make it inline assembly. Needed for IFUNC resolvers as otherwise Cpu::runtimeFeatures() could be a cross-SO call that crashes on Clang and older GCC. Everything seemed to work in the localized case of CpuTest alone, but as soon as I started using IFUNCs in other libraries, only GCC 11 continued to work and GCC 4.8 and Clang started to crashing inside _dl_somewhere(). Took me a while to discover also, due to *all the other* issues that I encountered. Because it's essential for the header to be small, it doesn't drag in 37kLOC of (!!!) headers and instead uses inline asssembly. Looking at the diff, should have done that right from the start instead of cursing at pointless compiler differences. Six lines less, with no functional change, and no header dependencies. --- src/Corrade/Cpu.cpp | 144 ++------------------------------------------ src/Corrade/Cpu.h | 139 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 141 insertions(+), 142 deletions(-) diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index cf2e3bdb1..a858fc710 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -29,30 +29,6 @@ #include "Corrade/Containers/StringView.h" #include "Corrade/Utility/Debug.h" -#ifdef CORRADE_TARGET_X86 -#include -#ifdef CORRADE_TARGET_MSVC -#include -#include -#elif defined(CORRADE_TARGET_GCC) -#include -/* _xgetbv() defined since GCC 8.1, in , but I can't include - that one alone, have to include instead: - https://github.com/gcc-mirror/gcc/commit/b9bdd60b8792e2d3173ecfacd5c25aac894a94e5 - On Clang it got first defined in 3.5: - https://github.com/llvm/llvm-project/commit/854f7d34ec3d7c0559ffde536cfa88fad5419bf0 - but that's (look again) in Intrin.h (yes, uppercase), which got fixed to - be intrin.h in 3.9, however at least on Xcode 9.4 intrin.h has - `#include_next ` and there *just isn't* any other intrin.h so - it fails; then, looking at the commit history, the function changed - locations a few times, mostly as an attempt to fix compilation of V8 (?!?!) - and ... I have no patience for this anymore, so for Clang I'm using the same - inline assembly as for GCC 8.0 and earlier. */ -#if defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 801 -#include -#endif -#endif - /* ARM on Linux and Android with API level 18+ */ #elif defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) #include @@ -150,121 +126,10 @@ int appleSysctlByName(const char* name) { } #endif -#if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && ((defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(CORRADE_TARGET_APPLE))) -#if defined(CORRADE_TARGET_X86) && defined(CORRADE_TARGET_GCC) -/* GCC 10 complains otherwise, however 4.8 doesn't. Doing it via a function - attribute instead of passing -mxsave as detecting target architecture in - CMake is annoying at best. */ -__attribute__((__target__("xsave"))) -#endif +#if defined(CORRADE_TARGET_ARM) && ((defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(CORRADE_TARGET_APPLE)) Features runtimeFeatures() { - /* Use cpuid on x86 and GCC/Clang/MSVC */ - #if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) - union { - struct { - std::uint32_t ax, bx, cx, dx; - } e; - #ifdef CORRADE_TARGET_MSVC - /* MSVC's __cpuid() macro accepts an array instead of four pointers and - wants an int, not uint, for some reason */ - std::int32_t data[4]; - #endif - } cpuid{}; - - /* https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits */ - #ifdef CORRADE_TARGET_MSVC - __cpuid(cpuid.data, 1); - #elif defined(CORRADE_TARGET_GCC) - __get_cpuid(1, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx); - #endif - - unsigned int out = 0; - if(cpuid.e.dx & (1 << 26)) out |= TypeTraits::Index; - if(cpuid.e.cx & (1 << 0)) out |= TypeTraits::Index; - if(cpuid.e.cx & (1 << 9)) out |= TypeTraits::Index; - if(cpuid.e.cx & (1 << 19)) out |= TypeTraits::Index; - if(cpuid.e.cx & (1 << 20)) out |= TypeTraits::Index; - - /* https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits, - bit 5 says "ABM (lzcnt and popcnt)", but - https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation) - says that while LZCNT is advertised in the ABM CPUID bit, POPCNT is a - separate CPUID flag. Get POPCNT first, ABM later. */ - if(cpuid.e.cx & (1 << 23)) out |= TypeTraits::Index; - - /* AVX needs OS support checked, as the OS needs to be capable of saving - and restoring the expanded registers when switching contexts: - https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Operating_system_support */ - if((cpuid.e.cx & (1 << 27)) && /* XSAVE/XRESTORE CPU support */ - (cpuid.e.cx & (1 << 28))) /* AVX CPU support */ - { - /* XGETBV indicates that the registers will be properly saved and - restored by the OS: https://stackoverflow.com/a/22521619. The - _xgetbv() function is available on MSVC and since GCC 8.1 and - probably on Clang also but I don't care -- see the comment at the - #include above. */ - #if defined(CORRADE_TARGET_MSVC) || (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 801) - const bool available = (_xgetbv(0) & 0x6) == 0x6; - #else - /* https://github.com/vectorclass/version2/blob/ff7450acfad9d3a7c6825d92cfb782a42ccfa71f/instrset_detect.cpp#L30-L32 */ - std::uint32_t a, d; - __asm("xgetbv" : "=a"(a),"=d"(d) : "c"(0) : ); - const bool available = ((a|(std::uint64_t(d) << 32)) & 0x6) == 0x6; - #endif - - if(available) out |= TypeTraits::Index; - } - - /* If AVX is not supported, we don't check any following flags either */ - if(out & TypeTraits::Index) { - if(cpuid.e.cx & (1 << 29)) out |= TypeTraits::Index; - if(cpuid.e.cx & (1 << 12)) out |= TypeTraits::Index; - - /* https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features */ - #ifdef CORRADE_TARGET_MSVC - __cpuidex(cpuid.data, 7, 0); - /* __get_cpuid_count() is only since GCC 7.1 and Clang 5.0: - https://github.com/gcc-mirror/gcc/commit/2488ebe5ef1788616c2fbc61e05af09f0749ebbe - https://github.com/llvm/llvm-project/commit/f6e8408a116405d0cc25b506e8c5b60ab4ab7bcd - Furthermore, Clang 5.0 is 9.3 on Apple, 9.2 has only Clang 4: - https://en.wikipedia.org/wiki/Xcode#Toolchain_versions */ - #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 701) || (defined(CORRADE_TARGET_CLANG) && ((!defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__ >= 5) || (defined(CORRADE_TARGET_APPLE_CLANG) && __clang_major__*100 + __clang_minor__ >= 903))) - __get_cpuid_count(7, 0, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx); - /* Looking at the above commits, on older GCC, __get_cpuid(7) seems to - do what __get_cpuid_count(7, 0) does on new GCC, however that's only - since 6.3: - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 - Older Clang doesn't do any of that, and thus to avoid a - combinatorial explosion of test scenarios, we'll just directly use - the __cpuid assembly on both, which seems to be there since forever. - __get_cpuid[_count]() only does a bunch of bounds checking on top of - that anyway and if we didn't blow up on the assert above I HOPE we - should be fine to check for extended features level zero. For higher - levels (to check very recent AVX512 features) probably not but I - assume those would need very recent compilers anyway so when that - happens we can just pretend those don't exist on GCC 7.0 and - older. */ - #else - __cpuid_count(7, 0, cpuid.e.ax, cpuid.e.bx, cpuid.e.cx, cpuid.e.dx); - #endif - if(cpuid.e.bx & (1 << 3)) out |= TypeTraits::Index; - if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; - if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; - } - - /* And now the LZCNT bit, finally - https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits */ - #ifdef CORRADE_TARGET_MSVC - __cpuid(cpuid.data, 0x80000001); - #elif defined(CORRADE_TARGET_GCC) - __get_cpuid(0x80000001, &cpuid.e.ax, &cpuid.e.bx, &cpuid.e.cx, &cpuid.e.dx); - #endif - if(cpuid.e.cx & (1 << 5)) out |= TypeTraits::Index; - - return Features{out}; - /* Use getauxval() on ARM on Linux and Android */ - #elif defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) + #if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) /* People say getauxval() is "extremely slow": https://lemire.me/blog/2020/07/17/the-cost-of-runtime-dispatch/#comment-538459 Like, can anything be worse than reading and parsing the text from @@ -308,8 +173,9 @@ Features runtimeFeatures() { return Features{out}; - /* No other implementation at the moment. The function should not be even - defined here in that case -- it's inlined in the header instead. */ + /* No other (deinlined) implementation at the moment. The function should + not be even defined here in that case -- it's inlined in the header + instead, including the x86 implementation. */ #else #error #endif diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index cdad8c6a1..aef3069c2 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -36,6 +36,17 @@ #include "Corrade/Utility/Utility.h" #include "Corrade/Utility/visibility.h" +/* Because can't use inline assembly when targeting 64bit on MSVC, and because + and is just too damn heavy to be included in a + header. Declarations copied verbatim. Clang-cl doesn't like this (undefined + reference to __cpuidex), so using the GCC/Clang codepath on it instead. */ +#if defined(CORRADE_TARGET_MSVC) && !defined(CORRADE_TARGET_CLANG_CL) && defined(CORRADE_TARGET_X86) +extern "C" { + void __cpuidex(int[4], int, int); + unsigned __int64 __cdecl _xgetbv(unsigned int); +} +#endif + namespace Corrade { /** @@ -1423,8 +1434,12 @@ class Features { template friend constexpr Features features(); friend constexpr Features compiledFeatures(); #if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE)) - /* MSVC demands the export macro to be here as well */ - friend CORRADE_UTILITY_EXPORT Features runtimeFeatures(); + friend + #ifdef CORRADE_TARGET_ARM + /* MSVC demands the export macro to be here as well. Inlined on x86. */ + CORRADE_UTILITY_EXPORT + #endif + Features runtimeFeatures(); #endif #if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) friend Features Implementation::runtimeFeatures(unsigned long); @@ -1690,7 +1705,10 @@ default-constructed) @ref Features. @see @ref DefaultBase, @ref DefaultExtra, @ref Default */ #if (defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC))) || (defined(CORRADE_TARGET_ARM) && ((defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18)) || defined(CORRADE_TARGET_APPLE))) || defined(DOXYGEN_GENERATING_OUTPUT) -CORRADE_UTILITY_EXPORT Features runtimeFeatures(); +#ifdef CORRADE_TARGET_ARM +CORRADE_UTILITY_EXPORT /* Inlined on x86 at the very end of the header */ +#endif +Features runtimeFeatures(); #else constexpr Features runtimeFeatures() { return compiledFeatures(); } #endif @@ -2978,6 +2996,121 @@ preprocessor behavior would make this extremely tricky to implement. #define CORRADE_ENABLE(...) #endif +/* x86 CPUID implementation on GCC/Clang/MSVC. Has to be inlined in the header + because otherwise in the IFUNC scenario it may result in a cross-SO call + that's unsupported on Clang and older GCC, causing a crash in the dynamic + loader during early startup because it calls into a place that's not there + yet. + + Because casually including leads to 37+ kLOC (!!!), I go an + extra way and use inline assembly instead. Which, compared to using the + intrinsics --- funnily enough --- reduces the amount of cursing and lengthy + comments explaining compiler bugs and differences to an absolute minimum. + If any of the following misbehaves, please check Git history for the + original implementation. */ +#if defined(CORRADE_TARGET_X86) && (defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_GCC)) +namespace Implementation { + inline void cpuid(int data[4], int leaf, int count) { + /* What's in GCC's / Clang's cpuid.h. Clang-cl as well, as it doesn't + seem to know the __cpuidex() intrinsics. */ + #if defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) + #ifdef CORRADE_TARGET_32BIT + asm("cpuid": \ + "=a"(data[0]), "=b"(data[1]), "=c"(data[2]), "=d"(data[3]): \ + "0"(leaf), "2"(count)); + #else + /* Clang says "x86-64 uses %rbx as the base register", GCC says "%rbx + may be the PIC register", so probably important to preserve it or + some such? ¯\_(ツ)_/¯ */ + asm("xchgq %%rbx,%q1\n" \ + "cpuid\n" \ + "xchgq %%rbx,%q1": \ + "=a"(data[0]), "=b"(data[1]), "=c"(data[2]), "=d"(data[3]): \ + "0"(leaf), "2"(count)); + #endif + + /* Declared at the top of the file */ + #elif defined(CORRADE_TARGET_MSVC) + __cpuidex(data, leaf, count); + #else + #error + #endif + } +} + +inline Features runtimeFeatures() { + union { + struct { + unsigned int ax, bx, cx, dx; + } e; + int data[4]; + } cpuid{}; + + Implementation::cpuid(cpuid.data, 1, 0); + + /* https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits */ + unsigned int out = 0; + if(cpuid.e.dx & (1 << 26)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 0)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 9)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 19)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 20)) out |= TypeTraits::Index; + + /* https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits, + bit 5 says "ABM (lzcnt and popcnt)", but + https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set#ABM_(Advanced_Bit_Manipulation) + says that while LZCNT is advertised in the ABM CPUID bit, POPCNT is a + separate CPUID flag. Get POPCNT first, ABM later. */ + if(cpuid.e.cx & (1 << 23)) out |= TypeTraits::Index; + + /* AVX needs OS support checked, as the OS needs to be capable of saving + and restoring the expanded registers when switching contexts: + https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#Operating_system_support */ + if((cpuid.e.cx & (1 << 27)) && /* XSAVE/XRESTORE CPU support */ + (cpuid.e.cx & (1 << 28))) /* AVX CPU support */ + { + /* XGETBV indicates that the registers will be properly saved and + restored by the OS: https://stackoverflow.com/a/22521619. */ + + /* https://github.com/vectorclass/version2/blob/ff7450acfad9d3a7c6825d92cfb782a42ccfa71f/instrset_detect.cpp#L30-L32 + Clang-cl as well, as it doesn't seem to know the MSVC intrinsics. */ + #if defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) + unsigned int a, d; + __asm("xgetbv": "=a"(a), "=d"(d): "c"(0): ); + const unsigned long long xgetbv = a|(static_cast(d) << 32); + + /* Declared at the top of the file */ + #elif defined(CORRADE_TARGET_MSVC) + const unsigned long long xgetbv = _xgetbv(0); + #else + #error + #endif + + if((xgetbv & 0x06) == 0x6) + out |= TypeTraits::Index; + } + + /* If AVX is not supported, we don't check any following flags either */ + if(out & TypeTraits::Index) { + if(cpuid.e.cx & (1 << 29)) out |= TypeTraits::Index; + if(cpuid.e.cx & (1 << 12)) out |= TypeTraits::Index; + + /* https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features */ + Implementation::cpuid(cpuid.data, 7, 0); + if(cpuid.e.bx & (1 << 3)) out |= TypeTraits::Index; + if(cpuid.e.bx & (1 << 5)) out |= TypeTraits::Index; + if(cpuid.e.bx & (1 << 16)) out |= TypeTraits::Index; + } + + /* And now the LZCNT bit, finally + https://en.wikipedia.org/wiki/CPUID#EAX=80000001h:_Extended_Processor_Info_and_Feature_Bits */ + Implementation::cpuid(cpuid.data, 0x80000001, 0); + if(cpuid.e.cx & (1 << 5)) out |= TypeTraits::Index; + + return Features{out}; +} +#endif + } } From ac08aa2156701d217b8d60ba4888bbd7dce3b8fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 25 Jul 2022 21:22:37 +0200 Subject: [PATCH 43/79] Cpu: inline the internal ARM runtimeFeatures() query also. For the same reason, to make IFUNCs work. This one just smashes bits out of an externally provided value, so no crazy assembly rewrite needs to happen. --- src/Corrade/Cpu.cpp | 53 +++----------------------------------------- src/Corrade/Cpu.h | 54 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 51 deletions(-) diff --git a/src/Corrade/Cpu.cpp b/src/Corrade/Cpu.cpp index a858fc710..c782621d8 100644 --- a/src/Corrade/Cpu.cpp +++ b/src/Corrade/Cpu.cpp @@ -29,12 +29,11 @@ #include "Corrade/Containers/StringView.h" #include "Corrade/Utility/Debug.h" -/* ARM on Linux and Android with API level 18+ */ -#elif defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) -#include +/* getauxval() for ARM on Linux and Android with API level 18+ */ +#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) #include -/* ARM on macOS / iOS */ +/* sysctlbyname() for ARM on macOS / iOS */ #elif defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) #include #endif @@ -65,52 +64,6 @@ static_assert(sizeof(Cpu::NeonFp16) == 1, ""); static_assert(sizeof(Cpu::Simd128) == 1, ""); #endif -#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) -namespace Implementation { - -/* As getauxval() can't be called from within an ifunc resolver because there - it's too early for an external call, the value of AT_HWCAP is instead passed - to it from outside, on glibc 2.13+ and on Android API 30+: - https://github.com/bminor/glibc/commit/7520ff8c744a704ca39741c165a2360d63a4f47a - https://android.googlesource.com/platform/bionic/+/e949195f6489653ee3771535951ed06973246c3e/libc/include/sys/ifunc.h - Which means we need a variant of runtimeFeatures() that is able to operate - with a value fed from outside, which is then used inside such resolvers. - For simplicity this variant is available always and the public - Cpu::runtimeFeatures() just delegates to it. */ -/** @todo If AT_HWCAP2 or other bits are needed, it's passed to ifunc resolvers - only since glibc 2.30 (and Android API 30+, which is the same as before): - https://github.com/bminor/glibc/commit/2b8a3c86e7606cf1b0a997dad8af2d45ae8989c3 */ -Features runtimeFeatures(const unsigned long caps) { - unsigned int out = 0; - #ifdef CORRADE_TARGET_32BIT - if(caps & HWCAP_NEON) out |= TypeTraits::Index; - /* Since FMA is enabled by passing -mfpu=neon-vfpv4, I assume this is the - flag that corresponds to it. */ - if(caps & HWCAP_VFPv4) out |= TypeTraits::Index; - #else - /* On ARM64 NEON and NEON FMA is implicit. For extra security make use of - the CORRADE_TARGET_ defines (which should be always there). */ - out |= - #ifdef CORRADE_TARGET_NEON - TypeTraits::Index| - #endif - #ifdef CORRADE_TARGET_NEON_FMA - TypeTraits::Index| - #endif - 0; - /* The HWCAP flags are extremely cryptic. The only vague confirmation is in - a *commit message* to the kernel hwcaps file, FFS. The HWCAP_FPHP seems - to correspond to scalar FP16, so the other should be the vector one? - https://github.com/torvalds/linux/blame/master/arch/arm64/include/uapi/asm/hwcap.h - This one also isn't present on 32-bit, so I assume it's ARM64-only? */ - if(caps & HWCAP_ASIMDHP) out |= TypeTraits::Index; - #endif - return Features{out}; -} - -} -#endif - /* Helper for getting macOS / iOS ARM properties. Yep, it's stringly typed. */ #if defined(CORRADE_TARGET_ARM) && defined(CORRADE_TARGET_APPLE) namespace { diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index aef3069c2..8e37e760c 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -1308,7 +1308,9 @@ template constexpr T tag() { return T{Implementation::Init}; } #if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) namespace Implementation { - CORRADE_UTILITY_EXPORT Features runtimeFeatures(unsigned long caps); + /* Needed for a friend declaration, implementation is at the very end of + the header */ + Features runtimeFeatures(unsigned long caps); } #endif @@ -3111,6 +3113,56 @@ inline Features runtimeFeatures() { } #endif +/* ARM implementation on Linux and Android, inlined for the same reason as the + x86 variant above -- to make IFUNC work. As getauxval() can't be called from + within an ifunc resolver because there it's too early for an external call, + the value of AT_HWCAP is instead passed to it from outside, on glibc 2.13+ + and on Android API 30+: + https://github.com/bminor/glibc/commit/7520ff8c744a704ca39741c165a2360d63a4f47a + https://android.googlesource.com/platform/bionic/+/e949195f6489653ee3771535951ed06973246c3e/libc/include/sys/ifunc.h + Which means we need a variant of runtimeFeatures() that is able to operate + with a value fed from outside, which is then used inside such resolvers. A + nice consequence of that is that we don't need any other headers. + + The public Cpu::runtimeFeatures() is deinlined, calls getauxval() and passes + it into this function. */ +/** @todo If AT_HWCAP2 or other bits are needed, it's passed to ifunc resolvers + only since glibc 2.30 (and Android API 30+, which is the same as before): + https://github.com/bminor/glibc/commit/2b8a3c86e7606cf1b0a997dad8af2d45ae8989c3 */ +#if defined(CORRADE_TARGET_ARM) && defined(__linux__) && !(defined(CORRADE_TARGET_ANDROID) && __ANDROID_API__ < 18) +namespace Implementation { + inline Features runtimeFeatures(const unsigned long caps) { + unsigned int out = 0; + #ifdef CORRADE_TARGET_32BIT + if(caps & (1 << 12) /*HWCAP_NEON*/) out |= TypeTraits::Index; + /* Since FMA is enabled by passing -mfpu=neon-vfpv4, I assume this is + the flag that corresponds to it. */ + if(caps & (1 << 16) /*HWCAP_VFPv4*/) out |= TypeTraits::Index; + #else + /* On ARM64 NEON and NEON FMA is implicit. For extra security make use + of the CORRADE_TARGET_ defines (which should be always there). */ + out |= + #ifdef CORRADE_TARGET_NEON + TypeTraits::Index| + #endif + #ifdef CORRADE_TARGET_NEON_FMA + TypeTraits::Index| + #endif + 0; + /* The HWCAP flags are extremely cryptic. The only vague confirmation + is in a *commit message* to the kernel hwcaps file, FFS. The + HWCAP_FPHP seems to correspond to scalar FP16, so the other should + be the vector one? + https://github.com/torvalds/linux/blame/master/arch/arm64/include/uapi/asm/hwcap.h + This one also isn't present on 32-bit, so I assume it's + ARM64-only? */ + if(caps & (1 << 10) /*HWCAP_ASIMDHP*/) out |= TypeTraits::Index; + #endif + return Features{out}; + } +} +#endif + } } From b38c532db43cda1466e9a4544a5edd17cb491988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 25 Jul 2022 21:39:36 +0200 Subject: [PATCH 44/79] Cpu: give up on extra instruction sets on GCC 4.8. I was happy when I realized that if I enable a single target and then include an intrinsics header, the instruction set is then not tied to other targets enabled at the same time. I.e., if I do #pragma GCC push_options #pragma GCC target("avx") #include #pragma GCC pop_options #pragma GCC push_options #pragma GCC target("lzcnt") #include #pragma GCC pop_options (plus the __AVX__ and __LZCNT__ macro push/pop of course) instead of #pragma GCC push_options #pragma GCC target("avx") #pragma GCC target("lzcnt") #include // also grabs LZCNT #pragma GCC pop_options then I can use LZCNT without having to enable AVX as well. What I didn't know back then was that on GCC 4.8 this works the other way as well -- so now I cannot use LZCNT together with AVX, because the compiler somehow remembers that LZCNT can be used only if target is EXACTLY what was used at the point when the intrinsics header was included, instead of if it just CONTAINS that. Haha. Which makes it pretty useless in practice, since e.g. the String additions I have use LZCNT / TZCNT with either SSE2 or AVX2. Since SSE2 is implicit, that worked, but as soon I got to the AVX code path, it no longer did, failing with a linker error. The commit that makes all this unnecessary on GCC 4.9 is mostly just header updates, but also two minor code changes, and those changes seem to be *essential* for this to work. Meaning, I just have no way to make my use case work with GCC 4.8: https://github.com/gcc-mirror/gcc/commit/97db2bf7fb10e7eb2e8224e0471b56976f133843 While I could work around this with nasty ifdefs in user code, I feel like the more robust way is to just not have the CORRADE_ENABLE_ macros for the extra instruction sets there at all, thus avoiding any further suffering right from the start. For the "base" instruction sets it seems fine so far, most probably because there's some inheritance / dependency between them. I hope I wouldn't need to drop those too. --- src/Corrade/Cpu.h | 39 +++++++++--------- src/Corrade/Utility/IntrinsicsAvx.h | 61 +++------------------------- src/Corrade/Utility/IntrinsicsSse4.h | 20 ++------- 3 files changed, 29 insertions(+), 91 deletions(-) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 8e37e760c..997f22362 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -2278,7 +2278,9 @@ expands to @cpp __attribute__((__target__("popcnt"))) @ce, allowing use of the instructions inside a function annotated with this macro without having to specify `-mpopcnt` for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of intrinsics in any way. Not -defined on other compilers or architectures. +defined on GCC 4.8, as there it's not generally possible to enable it alongside +other instruction sets without running into linker errors. Not defined on other +compilers or architectures. As a special case, if @ref CORRADE_TARGET_POPCNT is defined (meaning POCNT is enabled for the whole compilation unit), this macro is defined as empty on all @@ -2303,7 +2305,7 @@ may need to specify it together with others. See #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_POPCNT #endif -#elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) +#elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 409) || defined(CORRADE_TARGET_CLANG) /* matches clang-cl */ #define CORRADE_ENABLE_POPCNT __attribute__((__target__("popcnt"))) #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_POPCNT "popcnt", @@ -2325,7 +2327,9 @@ nothing, as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE variants and POPCNT this macro is not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, BMI1, AVX and newer intrinsics are provided only if enabled on compiler command line. Not defined -on other compilers or architectures. +on GCC 4.8, as there it's not generally possible to enable it alongside +unrelated instruction sets without running into linker errors. Not defined on +other compilers or architectures. As a special case, if @ref CORRADE_TARGET_LZCNT is defined (meaning LZCNT is enabled for the whole compilation unit), this macro is defined as empty on all @@ -2350,20 +2354,11 @@ may need to specify it together with others. See #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_LZCNT #endif -#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ -/* GCC 4.8 needs also -mabm. This option is not needed since 4.9 and is also - unrecognized on Clang. */ -#if defined(CORRADE_TARGET_CLANG) || __GNUC__*100 + __GNUC_MINOR__ >= 409 +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ #define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt"))) #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_LZCNT "lzcnt", #endif -#else -#define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt,abm"))) -#ifdef CORRADE_TARGET_GCC -#define _CORRADE_ENABLE_LZCNT "lzcnt,abm", -#endif -#endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, still present in Jul 2022, meaning we can only use these if __LZCNT__ is defined. Funnily enough the older headers don't have this on their own, only @@ -2385,7 +2380,9 @@ specify `-mbmi` for the whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE variants and POPCNT this macro is not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there LZCNT, BMI1, AVX and newer -intrinsics are provided only if enabled on compiler command line. Not defined on +intrinsics are provided only if enabled on compiler command line. Not defined +on GCC 4.8, as there it's not generally possible to enable it alongside +unrelated instruction sets without running into linker errors. Not defined on other compilers or architectures. As a special case, if @ref CORRADE_TARGET_BMI1 is defined (meaning BMI1 is @@ -2411,7 +2408,7 @@ may need to specify it together with others. See #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_BMI1 #endif -#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ #define CORRADE_ENABLE_BMI1 __attribute__((__target__("bmi"))) #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_BMI1 "bmi", @@ -2490,7 +2487,9 @@ compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE variants this macro is not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer intrinsics are provided only if enabled on compiler command line. Not defined -on other compilers or architectures. +on GCC 4.8, as there it's not generally possible to enable it alongside other +instruction sets without running into linker errors. Not defined on other +compilers or architectures. As a special case, if @ref CORRADE_TARGET_AVX_F16C is present (meaning AVX F16C is enabled for the whole compilation unit), this macro is defined as empty on @@ -2516,7 +2515,7 @@ information and usage example. #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_AVX_F16C #endif -#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ /* The -mf16c option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ #define CORRADE_ENABLE_AVX_F16C __attribute__((__target__("f16c"))) @@ -2544,7 +2543,9 @@ whole compilation unit. On x86 MSVC expands to nothing, as the compiler doesn't restrict use of intrinsics in any way. Unlike the SSE variants this macro is not defined on @ref CORRADE_TARGET_CLANG_CL "clang-cl", as there AVX and newer intrinsics are provided only if enabled on compiler command line. Not defined -on other compilers or architectures. +on GCC 4.8, as there it's not generally possible to enable it alongside other +instruction sets without running into linker errors. Not defined on other +compilers or architectures. As a special case, if @ref CORRADE_TARGET_AVX_FMA is present (meaning AVX with FMA is enabled for the whole compilation unit), this macro is defined as empty @@ -2570,7 +2571,7 @@ information and usage example. #ifdef CORRADE_TARGET_GCC #define _CORRADE_ENABLE_AVX_FMA #endif -#elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ +#elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ /* The -mfma option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ #define CORRADE_ENABLE_AVX_FMA __attribute__((__target__("fma"))) diff --git a/src/Corrade/Utility/IntrinsicsAvx.h b/src/Corrade/Utility/IntrinsicsAvx.h index 236e4c045..1f27acc3b 100644 --- a/src/Corrade/Utility/IntrinsicsAvx.h +++ b/src/Corrade/Utility/IntrinsicsAvx.h @@ -32,8 +32,11 @@ Equivalent to @cpp #include @ce on most compilers except for GCC 4.8, where it contains an additional workaround to make the instructions available with just the @ref CORRADE_ENABLE_AVX, @ref CORRADE_ENABLE_AVX_F16C, @ref CORRADE_ENABLE_AVX_FMA or @ref CORRADE_ENABLE_AVX2 function attributes -instead of having to specify `-mlzcnt`, `-mbmi`, `-mavx`, `-mf16c`, `-mfma` or -`-mavx2` for the whole compilation unit. +instead of having to specify `-mavx` or `-mavx2` for the whole compilation +unit. This however can't reliably be done for `-mlzcnt`, `-mbmi`, `-mf16c` or +`-mfma` because then it could not be freely combined with other instruction +sets, only used alone. You have to enable these instructions globally in order +to use them on GCC 4.8. As AVX-512 is supported only since GCC 4.9, which doesn't need this workaround, it's not handled here. @@ -90,57 +93,3 @@ it's not handled here. #pragma pop_macro("__AVX2__") #pragma GCC pop_options #endif - -#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 -#pragma GCC push_options -#pragma GCC target("f16c") -#pragma push_macro("__F16C__") -#ifndef __F16C__ -#define __F16C__ -#endif -#include -#pragma pop_macro("__F16C__") -#pragma GCC pop_options -#endif - -#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 -#pragma GCC push_options -#pragma GCC target("fma") -#pragma push_macro("__FMA__") -#ifndef __FMA__ -#define __FMA__ -#endif -#include -#pragma pop_macro("__FMA__") -#pragma GCC pop_options -#endif - -#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 -#pragma GCC push_options -#pragma GCC target("lzcnt") -#pragma GCC target("abm") /* GCC 4.8 needs this, 4.9 not; urecognized on Clang */ -#pragma push_macro("__LZCNT__") -#pragma push_macro("__ABM__") -#ifndef __ABM__ -#define __ABM__ -#endif -#ifndef __LZCNT__ -#define __LZCNT__ -#endif -#include -#pragma pop_macro("__ABM__") -#pragma pop_macro("__LZCNT__") -#pragma GCC pop_options -#endif - -#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 -#pragma GCC push_options -#pragma GCC target("bmi") -#pragma push_macro("__BMI__") -#ifndef __BMI__ -#define __BMI__ -#endif -#include -#pragma pop_macro("__BMI__") -#pragma GCC pop_options -#endif diff --git a/src/Corrade/Utility/IntrinsicsSse4.h b/src/Corrade/Utility/IntrinsicsSse4.h index a35a93507..c6eae1c63 100644 --- a/src/Corrade/Utility/IntrinsicsSse4.h +++ b/src/Corrade/Utility/IntrinsicsSse4.h @@ -34,7 +34,10 @@ Equivalent to @cpp #include @ce and - GCC 4.8, where it contains an additional workaround to make the instructions available with just the @ref CORRADE_ENABLE_SSE41 and @ref CORRADE_ENABLE_SSE42 function attributes instead of having to specify - `-msse4.1`, `-msse4.2` or `-mpopcnt` for the whole compilation unit. + `-msse4.1` or `-msse4.2` for the whole compilation unit. This however can't + reliably be done for `-mpopcnt` because then it could not be freely + combined with other instruction sets, only used alone. You have to enable + these instructions globally in order to use them on GCC 4.8. - Clang < 7, where `__POPCNT__` has to be explicitly defined in order to access the POPCNT instruction @@ -90,18 +93,3 @@ together, unlike with other SSE variants. #if defined(CORRADE_TARGET_CLANG) && __clang_major__ < 7 #pragma pop_macro("__POPCNT__") #endif - -/* Unlike with SSE4.1 / SSE4.2 there's no weird interaction with the popcnt - intrinsics, so it could be even in the above block. But for extra safety and - consistency with AVX it's separate. */ -#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) && __GNUC__*100 + __GNUC_MINOR__ < 409 -#pragma GCC push_options -#pragma GCC target("popcnt") -#pragma push_macro("__POPCNT__") -#ifndef __POPCNT__ -#define __POPCNT__ -#endif -#include -#pragma pop_macro("__POPCNT__") -#pragma GCC pop_options -#endif From cf31a989a00291eccd02103cbbbc631251b9e305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 17:25:51 +0200 Subject: [PATCH 45/79] Cpu: multiple CORRADE_ENABLE_ macros don't work on Clang < 8 either. Argh! What other curses are still there? --- doc/corrade-developers.dox | 4 +- src/Corrade/Cpu.h | 88 +++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/doc/corrade-developers.dox b/doc/corrade-developers.dox index 2bc0fbea7..57e361205 100644 --- a/doc/corrade-developers.dox +++ b/doc/corrade-developers.dox @@ -535,8 +535,8 @@ inverse. - Annotated with the "function variant" of the macro (thus @cpp CORRADE_ENABLE(FOO) @ce) and also with an @cpp #ifdef @ce for the macro around. The function variant should be used in order to test both - the `_CORRADE_ENABLE_FOO` macro (on GCC) and `CORRADE_ENABLE_FOO` (on - Clang and elsewhere). + the `_CORRADE_ENABLE_FOO` macro (on Clang before version 8 and GCC) and + `CORRADE_ENABLE_FOO` (elsewhere). - Ideally using just that instruction alone (such as is the case with `POPCNT`), if not then try to use an instruction set that's implied by it so it doesn't need a second `CORRADE_ENABLE_*`. Clearly mark which diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 997f22362..463a73cd3 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -2074,12 +2074,12 @@ more information and usage example. */ #if defined(CORRADE_TARGET_SSE2) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE2 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE2 #endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE_SSE2 __attribute__((__target__("sse2"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE2 "sse2", #endif #elif defined(CORRADE_TARGET_MSVC) @@ -2117,14 +2117,14 @@ Superset of @ref CORRADE_ENABLE_SSE2, implied by @ref CORRADE_ENABLE_SSSE3. See */ #if defined(CORRADE_TARGET_SSE3) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE3 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE3 #endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -msse3 option implies -msse2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse3`) */ #define CORRADE_ENABLE_SSE3 __attribute__((__target__("sse3"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE3 "sse3", #endif #elif defined(CORRADE_TARGET_MSVC) @@ -2162,14 +2162,14 @@ Superset of @ref CORRADE_ENABLE_SSE3, implied by @ref CORRADE_ENABLE_SSE41. See */ #if defined(CORRADE_TARGET_SSSE3) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSSE3 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSSE3 #endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) /* The -mssse3 option implies -msse2 -msse3 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mssse3`) */ #define CORRADE_ENABLE_SSSE3 __attribute__((__target__("ssse3"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSSE3 "ssse3", #endif #elif defined(CORRADE_TARGET_MSVC) @@ -2207,7 +2207,7 @@ See @ref Cpu-usage-target-attributes for more information and usage example. */ #if defined(CORRADE_TARGET_SSE41) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE41 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE41 #endif #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 409) || defined(CORRADE_TARGET_CLANG) /* also matches clang-cl */ @@ -2215,7 +2215,7 @@ See @ref Cpu-usage-target-attributes for more information and usage example. no need to specify those as well (verified with `echo | gcc -dM -E - -msse4.1`) */ #define CORRADE_ENABLE_SSE41 __attribute__((__target__("sse4.1"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE41 "sse4.1", #endif #elif defined(CORRADE_TARGET_MSVC) @@ -2253,7 +2253,7 @@ Superset of @ref CORRADE_ENABLE_SSE41, implied by @ref CORRADE_ENABLE_AVX. See */ #if defined(CORRADE_TARGET_SSE42) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SSE42 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE42 #endif #elif defined(CORRADE_TARGET_GCC) || defined(CORRADE_TARGET_CLANG_CL) @@ -2261,7 +2261,7 @@ Superset of @ref CORRADE_ENABLE_SSE41, implied by @ref CORRADE_ENABLE_AVX. See Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -msse4.2`) */ #define CORRADE_ENABLE_SSE42 __attribute__((__target__("sse4.2"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SSE42 "sse4.2", #endif #elif defined(CORRADE_TARGET_MSVC) @@ -2302,12 +2302,12 @@ may need to specify it together with others. See */ #if defined(CORRADE_TARGET_POPCNT) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_POPCNT -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_POPCNT #endif #elif (defined(CORRADE_TARGET_GCC) && __GNUC__*100 + __GNUC_MINOR__ >= 409) || defined(CORRADE_TARGET_CLANG) /* matches clang-cl */ #define CORRADE_ENABLE_POPCNT __attribute__((__target__("popcnt"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_POPCNT "popcnt", #endif #elif defined(CORRADE_TARGET_MSVC) @@ -2351,12 +2351,12 @@ may need to specify it together with others. See */ #if defined(CORRADE_TARGET_LZCNT) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_LZCNT -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_LZCNT #endif #elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ #define CORRADE_ENABLE_LZCNT __attribute__((__target__("lzcnt"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_LZCNT "lzcnt", #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, @@ -2405,12 +2405,12 @@ may need to specify it together with others. See */ #if defined(CORRADE_TARGET_BMI1) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_BMI1 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_BMI1 #endif #elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ #define CORRADE_ENABLE_BMI1 __attribute__((__target__("bmi"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_BMI1 "bmi", #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, @@ -2455,7 +2455,7 @@ Superset of @ref CORRADE_ENABLE_SSE42, implied by @ref CORRADE_ENABLE_AVX2. See */ #if defined(CORRADE_TARGET_AVX) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX #endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ @@ -2463,7 +2463,7 @@ Superset of @ref CORRADE_ENABLE_SSE42, implied by @ref CORRADE_ENABLE_AVX2. See and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx`) */ #define CORRADE_ENABLE_AVX __attribute__((__target__("avx"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX "avx", #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, @@ -2512,14 +2512,14 @@ information and usage example. */ #if defined(CORRADE_TARGET_AVX_F16C) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX_F16C -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX_F16C #endif #elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ /* The -mf16c option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ #define CORRADE_ENABLE_AVX_F16C __attribute__((__target__("f16c"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX_F16C "f16c", #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, @@ -2568,14 +2568,14 @@ information and usage example. */ #if defined(CORRADE_TARGET_AVX_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX_FMA -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX_FMA #endif #elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ /* The -mfma option implies -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mavx on both GCC and Clang (verified with `echo | gcc -dM -E - -mf16c`) */ #define CORRADE_ENABLE_AVX_FMA __attribute__((__target__("fma"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX_FMA "fma", #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, @@ -2621,7 +2621,7 @@ See @ref Cpu-usage-target-attributes for more information and usage example. */ #if defined(CORRADE_TARGET_AVX2) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX2 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX2 #endif #elif defined(CORRADE_TARGET_GCC) /* does not match clang-cl */ @@ -2629,7 +2629,7 @@ See @ref Cpu-usage-target-attributes for more information and usage example. both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx2`) */ #define CORRADE_ENABLE_AVX2 __attribute__((__target__("avx2"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX2 "avx2", #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, @@ -2667,7 +2667,7 @@ more information and usage example. */ #if defined(CORRADE_TARGET_AVX512F) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_AVX512F -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX512F #endif #elif defined(CORRADE_TARGET_GCC) && (__GNUC__*100 + __GNUC_MINOR__ >= 409 || defined(CORRADE_TARGET_CLANG)) /* does not match clang-cl */ @@ -2675,7 +2675,7 @@ more information and usage example. -mavx2 on both GCC and Clang, so no need to specify those as well (verified with `echo | gcc -dM -E - -mavx512f`) */ #define CORRADE_ENABLE_AVX512F __attribute__((__target__("avx512f"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_AVX512F "avx512f", #endif /* https://github.com/llvm/llvm-project/commit/379a1952b37247975d2df8d23498675c9c8cc730, @@ -2714,7 +2714,7 @@ for more information and usage example. */ #if defined(CORRADE_TARGET_NEON) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_NEON #endif /* https://github.com/android/ndk/issues/1066 is the only reported (and @@ -2724,7 +2724,7 @@ for more information and usage example. ditched GCC, where this works properly. */ #elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) #define CORRADE_ENABLE_NEON __attribute__((__target__("fpu=neon"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_NEON "fpu=neon", #endif #elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) @@ -2756,13 +2756,13 @@ See @ref Cpu-usage-target-attributes for more information and usage example. */ #if defined(CORRADE_TARGET_NEON_FMA) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON_FMA -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_NEON_FMA #endif /* See CORRADE_ENABLE_NEON above for details about Clang */ #elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) #define CORRADE_ENABLE_NEON_FMA __attribute__((__target__("fpu=neon-vfpv4"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_NEON_FMA "fpu=neon-vfpv4", #endif #elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) @@ -2793,13 +2793,13 @@ for more information and usage example. */ #if defined(CORRADE_TARGET_NEON_FP16) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_NEON_FP16 -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_NEON_FP16 #endif /* See CORRADE_ENABLE_NEON above for details about Clang */ #elif defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) #define CORRADE_ENABLE_NEON_FP16 __attribute__((__target__("arch=armv8.2-a+fp16"))) -#ifdef CORRADE_TARGET_GCC +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_NEON_FP16 "arch=armv8.2-a+fp16", #endif #elif defined(CORRADE_TARGET_MSVC) || defined(DOXYGEN_GENERATING_OUTPUT) @@ -2830,15 +2830,16 @@ See @ref Cpu-usage-target-attributes for more information and usage example. */ #if defined(CORRADE_TARGET_SIMD128) || defined(DOXYGEN_GENERATING_OUTPUT) #define CORRADE_ENABLE_SIMD128 -#ifdef CORRADE_TARGET_GCC /* Future-proofing, GCC has no WASM support ATM */ +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_SIMD128 #endif #endif #endif #ifndef DOXYGEN_GENERATING_OUTPUT -/* GCC treats __attribute__((target("foo"))) __attribute__((target("bar"))) as - if only "bar" was specified, thus it's not possible to just put several +/* GCC and Clang before version 8 treat + __attribute__((target("foo"))) __attribute__((target("bar"))) + as if only "bar" was specified, thus it's not possible to just put several CORRADE_ENABLE_ macros after each other. Instead, the only accepted form is __attribute__((target("foo,bar"))). Fortunately, string literal concatenation works here, thus with some extra macro trickery we can produce @@ -2849,7 +2850,7 @@ See @ref Cpu-usage-target-attributes for more information and usage example. comma after the string literal (thus "foo",), which causes the empty macros be filtered out when passed one after another (without commas) from _CORRADE_ENABLEn to _CORRADE_ENABLE_CONCATENATE(). */ -#if defined(CORRADE_TARGET_GCC) && !defined(CORRADE_TARGET_CLANG) +#if defined(CORRADE_TARGET_GCC) && (!defined(CORRADE_TARGET_CLANG) || __clang_major__ < 8) #define _CORRADE_ENABLE_CONCATENATE0(unused) #define _CORRADE_ENABLE_CONCATENATE1(v0, unused) \ __attribute__((__target__(v0))) @@ -2985,13 +2986,14 @@ effectively enabling given combination. For the macro to work, all the common usage pattern is thus in combination with an @cpp #ifdef @ce. See @ref Cpu-usage-target-attributes for more information and an example. -When multiple `CORRADE_ENABLE_*` macros are specified one after another, GCC -would pick only the last specified, ignoring the others. For it, the macro -expands into a single combined @cpp __attribute__((__target__(...))) @ce -attribute. For other compilers except MSVC it's just a shorthand for multiple -`CORRADE_ENABLE_*` macros one after another. On MSVC expands to nothing --- -there the functions aren't annotated in anyway and moreover the default -preprocessor behavior would make this extremely tricky to implement. +When multiple `CORRADE_ENABLE_*` macros are specified one after another, Clang +before version 8 and GCC would pick only the last specified, ignoring the +others. There the macro expands into a single combined +@cpp __attribute__((__target__(...))) @ce attribute. For other compilers except +MSVC it's just a shorthand for multiple `CORRADE_ENABLE_*` macros one after +another. On MSVC expands to nothing --- there the functions aren't annotated in +anyway and moreover the default preprocessor behavior would make this extremely +tricky to implement. */ #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE(...) _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_ENABLE8, _CORRADE_ENABLE7, _CORRADE_ENABLE6, _CORRADE_ENABLE5, _CORRADE_ENABLE4, _CORRADE_ENABLE3, _CORRADE_ENABLE2, _CORRADE_ENABLE1, )(__VA_ARGS__) From 22613a6c8266f18905df294b52043e2a179c2e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 17:26:34 +0200 Subject: [PATCH 46/79] Cpu: older Clang also doesn't recognize lambda attributes. Ah well, so gotta put them on both places. --- doc/snippets/Corrade.cpp | 2 +- src/Corrade/Cpu.h | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/snippets/Corrade.cpp b/doc/snippets/Corrade.cpp index 77e6e1ec5..dd5c11070 100644 --- a/doc/snippets/Corrade.cpp +++ b/doc/snippets/Corrade.cpp @@ -107,7 +107,7 @@ namespace Baz { TransformT transformImplementation(Cpu::Avx2T); /* [Cpu-usage-automatic-runtime-dispatch-target-attributes] */ #ifdef CORRADE_ENABLE_AVX2 -TransformT transformImplementation(Cpu::Avx2T) { +CORRADE_ENABLE_AVX2 TransformT transformImplementation(Cpu::Avx2T) { return [](Containers::ArrayView data) CORRADE_ENABLE_AVX2 { DOXYGEN_ELLIPSIS(static_cast(data);) }; } #endif diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index 463a73cd3..e703a518c 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -308,10 +308,13 @@ actual arguments: @par Instruction enabling macros An important difference with the @ref CORRADE_ENABLE_SSE2 "CORRADE_ENABLE_*" - macros is that they now have to go directly next to the lambda as GCC + macros is that they now have to go also directly next to the lambda as GCC [currently doesn't propagate the attributes](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80439) - from the wrapper function to the nested lambda. The above AVX variant would - look like this with relevant macros added: + from the wrapper function to the nested lambda. To make matters worse, + older versions of Clang suffer from the inverse problem and ignore lambda + attributes, so you have to specify them on both the lambda and the wrapper + function. The above AVX variant would look like this with relevant macros + added: @par @snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-target-attributes From 17713aa12bfda128599a2d029b6364f0b02b7a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 6 Jul 2022 15:39:21 +0200 Subject: [PATCH 47/79] Containers: partially inline StringView::find*() and related APIs. A middle ground between inlining everything like STL does, causing excessive compile times, and inlining nothing, hurting optimization possibilities. - Since find() and trimmed() return a StringView, it's vital to have them inlined, to allow the compiler to optimize out the return value. Especially on Windows, where returning more than 64 bits is not free: https://aras-p.info/blog/2022/05/12/speeding-up-blender-obj-import/ - On the other hand, the low-level helpers that look for bytes in raw ranges of memory, are likely to significantly expand in size as they get rewritten with actual architecture-specific optimizations in mind. Right now, especially the find*Any*() variants, are quite shitty, calling memchr() in a loop, so this mainly avoids having #include in the header. Which is also a small win, but probably something to eventually revisit. - Lastly, as those low-level helpers get optimized, they might be useful in other low-level parts of the codebase such as the JSON parser, so having them easily accessible without additional overhead from a StringView is a win as well. Measuring the impact with `g++ -std=c++11 -E -P`, this increased the size of StringView.h from 4328 lines to 4392, with -DNDEBUG then from 2281 to 2344 lines (as Debug.h and is not pulled in), with 1.7k lines being from . Not too bad -- is still 12.5k lines. OTOH, is just 240, so it possibly might make sense to just pull it in as well and make operator== and construction with implicit length inline as well. --- src/Corrade/Containers/StringView.cpp | 111 +++--------------------- src/Corrade/Containers/StringView.h | 116 +++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 103 deletions(-) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 3160897e0..09b068865 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -97,9 +97,9 @@ template Array> BasicStringView::splitWithoutEmpt return parts; } -namespace { +namespace Implementation { -inline const char* find(const char* data, const std::size_t size, const char* const substring, const std::size_t substringSize) { +const char* stringFindString(const char* data, const std::size_t size, const char* const substring, const std::size_t substringSize) { /* If the substring is not larger than the string we search in */ if(substringSize <= size) { /* If these are both empty (substringSize <= size, so it's also 0), @@ -121,7 +121,7 @@ inline const char* find(const char* data, const std::size_t size, const char* co return {}; } -inline const char* findLast(const char* const data, const std::size_t size, const char* const substring, const std::size_t substringSize) { +const char* stringFindLastString(const char* const data, const std::size_t size, const char* const substring, const std::size_t substringSize) { /* If the substring is not larger than the string we search in */ if(substringSize <= size) { /* If these are both empty (substringSize <= size, so it's also 0), @@ -143,14 +143,14 @@ inline const char* findLast(const char* const data, const std::size_t size, cons return {}; } -inline const char* find(const char* data, const std::size_t size, const char character) { +const char* stringFindCharacter(const char* data, const std::size_t size, const char character) { /* Making a utility function because yet again I'm not sure if null pointers are allowed and cppreference says nothing about that, so in case this needs to be patched it's better to have it in a single place */ return static_cast(std::memchr(data, character, size)); } -inline const char* findLast(const char* const data, const std::size_t size, const char character) { +const char* stringFindLastCharacter(const char* const data, const std::size_t size, const char character) { /* Linux has a memrchr() function but other OSes not. So let's just do it myself, that way I also don't need to worry about null pointers being allowed or not ... haha, well, except that if data is nullptr, @@ -181,7 +181,7 @@ inline const char* findLast(const char* const data, const std::size_t size, cons std::find_first_of() because I doubt STL implementations explicitly optimize for that case. Yes, std::string::find_first_of() probably would have that, but I'd first need to allocate to make use of that and FUCK NO. */ -inline const char* findAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { +const char* stringFindAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data, *end = data + size; i != end; ++i) if(std::memchr(characters, *i, characterCount)) return i; return {}; @@ -190,19 +190,19 @@ inline const char* findAny(const char* const data, const std::size_t size, const /* Variants of the above. Not sure if those even have any vaguely corresponding C lib API. Probably not. */ -inline const char* findLastAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { +const char* stringFindLastAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data + size; i != data; --i) if(std::memchr(characters, *(i - 1), characterCount)) return i - 1; return {}; } -inline const char* findNotAny(const char* const data, const std::size_t size, const char* const characters, std::size_t characterCount) { +const char* stringFindNotAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data, *end = data + size; i != end; ++i) if(!std::memchr(characters, *i, characterCount)) return i; return {}; } -inline const char* findLastNotAny(const char* const data, const size_t size, const char* const characters, std::size_t characterCount) { +const char* stringFindLastNotAny(const char* const data, const std::size_t size, const char* const characters, const std::size_t characterCount) { for(const char* i = data + size; i != data; --i) if(!std::memchr(characters, *(i - 1), characterCount)) return i - 1; return {}; @@ -218,7 +218,7 @@ template Array> BasicStringView::splitOnAnyWithou T* const end = _data + size(); while(oldpos < end) { - if(T* const pos = const_cast(Containers::findAny(oldpos, end - oldpos, characters, characterCount))) { + if(T* const pos = const_cast(Implementation::stringFindAny(oldpos, end - oldpos, characters, characterCount))) { if(pos != oldpos) arrayAppend(parts, slice(oldpos, pos)); oldpos = pos + 1; @@ -408,10 +408,6 @@ template BasicStringView BasicStringView::exceptSuffix(const Stri return exceptSuffix(suffix.size()); } -template BasicStringView BasicStringView::trimmed(const StringView characters) const { - return trimmedPrefix(characters).trimmedSuffix(characters); -} - template BasicStringView BasicStringView::trimmed() const { #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) || _MSC_VER >= 1930 /* MSVC 2022 works */ return trimmed(Whitespace); @@ -421,12 +417,6 @@ template BasicStringView BasicStringView::trimmed() const { #endif } -template BasicStringView BasicStringView::trimmedPrefix(const StringView characters) const { - const std::size_t size = this->size(); - T* const found = const_cast(findNotAny(_data, size, characters._data, characters.size())); - return suffix(found ? found : _data + size); -} - template BasicStringView BasicStringView::trimmedPrefix() const { #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) || _MSC_VER >= 1930 /* MSVC 2022 works */ return trimmedPrefix(Whitespace); @@ -436,11 +426,6 @@ template BasicStringView BasicStringView::trimmedPrefix() const { #endif } -template BasicStringView BasicStringView::trimmedSuffix(const StringView characters) const { - T* const found = const_cast(findLastNotAny(_data, size(), characters._data, characters.size())); - return prefix(found ? found + 1 : _data); -} - template BasicStringView BasicStringView::trimmedSuffix() const { #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) || _MSC_VER >= 1930 /* MSVC 2022 works */ return trimmedSuffix(Whitespace); @@ -450,82 +435,6 @@ template BasicStringView BasicStringView::trimmedSuffix() const { #endif } -template BasicStringView BasicStringView::findOr(const StringView substring, T* const fail) const { - /* Cache the getters to speed up debug builds */ - const std::size_t substringSize = substring.size(); - if(const char* const found = Containers::find(_data, size(), substring._data, substringSize)) - return slice(const_cast(found), const_cast(found + substringSize)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findOr(const char character, T* const fail) const { - if(const char* const found = Containers::find(_data, size(), character)) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findLastOr(const StringView substring, T* const fail) const { - /* Cache the getters to speed up debug builds */ - const std::size_t substringSize = substring.size(); - if(const char* const found = Containers::findLast(_data, size(), substring._data, substringSize)) - return slice(const_cast(found), const_cast(found + substringSize)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findLastOr(const char character, T* const fail) const { - if(const char* const found = Containers::findLast(_data, size(), character)) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template bool BasicStringView::contains(const StringView substring) const { - return Containers::find(_data, size(), substring._data, substring.size()); -} - -template bool BasicStringView::contains(const char character) const { - return Containers::find(_data, size(), character); -} - -template BasicStringView BasicStringView::findAnyOr(const StringView characters, T* const fail) const { - if(const char* const found = Containers::findAny(_data, size(), characters._data, characters.size())) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template BasicStringView BasicStringView::findLastAnyOr(const StringView characters, T* const fail) const { - if(const char* const found = Containers::findLastAny(_data, size(), characters._data, characters.size())) - return slice(const_cast(found), const_cast(found + 1)); - - /* Using an internal assert-less constructor, the public constructor - asserts would be redundant. Since it's a zero-sized view, it doesn't - really make sense to try to preserve any flags. */ - return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; -} - -template bool BasicStringView::containsAny(const StringView characters) const { - return Containers::findAny(_data, size(), characters._data, characters.size()); -} - #ifndef DOXYGEN_GENERATING_OUTPUT template class /* GCC needs the export macro on the class definition (and here it warns diff --git a/src/Corrade/Containers/StringView.h b/src/Corrade/Containers/StringView.h index e2347bf1a..a3de4e44f 100644 --- a/src/Corrade/Containers/StringView.h +++ b/src/Corrade/Containers/StringView.h @@ -269,7 +269,15 @@ when you deal with unordered containers. @experimental */ /* All member functions are const because the view doesn't own the data */ -template class CORRADE_UTILITY_EXPORT BasicStringView { +template class +#ifndef CORRADE_TARGET_MSVC +/* If it's here, MSVC complains that the out-of-class inline functions have a + definition while being dllimport'd. If I remove it, GCC then complains that + the export in StringView.cpp is ignored as the type is already defined, + proceeding with a linker error. */ +CORRADE_UTILITY_EXPORT +#endif +BasicStringView { public: /** * @brief Default constructor @@ -796,7 +804,9 @@ template class CORRADE_UTILITY_EXPORT BasicStringView { * @see @ref trimmed() const, @ref trimmedPrefix(StringView) const, * @ref trimmedSuffix(StringView) const */ - BasicStringView trimmed(StringView characters) const; + BasicStringView trimmed(StringView characters) const { + return trimmedPrefix(characters).trimmedSuffix(characters); + } /** * @brief View with whitespace trimmed from prefix and suffix @@ -1254,6 +1264,108 @@ template constexpr BasicStringView BasicStringView::slice(const s nullptr}; } +namespace Implementation { + +/* Making naming unique in order to prepare for these being function pointers + (that can't be overloaded) */ +CORRADE_UTILITY_EXPORT const char* stringFindString(const char* data, std::size_t size, const char* substring, std::size_t substringSize); +CORRADE_UTILITY_EXPORT const char* stringFindLastString(const char* data, std::size_t size, const char* substring, std::size_t substringSize); +CORRADE_UTILITY_EXPORT const char* stringFindCharacter(const char* data, std::size_t size, char character); +CORRADE_UTILITY_EXPORT const char* stringFindLastCharacter(const char* data, std::size_t size, char character); +CORRADE_UTILITY_EXPORT const char* stringFindAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); +CORRADE_UTILITY_EXPORT const char* stringFindLastAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); +CORRADE_UTILITY_EXPORT const char* stringFindNotAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); +CORRADE_UTILITY_EXPORT const char* stringFindLastNotAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); + +} + +template inline BasicStringView BasicStringView::trimmedPrefix(const StringView characters) const { + const std::size_t size = this->size(); + T* const found = const_cast(Implementation::stringFindNotAny(_data, size, characters._data, characters.size())); + return suffix(found ? found : _data + size); +} + +template inline BasicStringView BasicStringView::trimmedSuffix(const StringView characters) const { + T* const found = const_cast(Implementation::stringFindLastNotAny(_data, size(), characters._data, characters.size())); + return prefix(found ? found + 1 : _data); +} + +template inline BasicStringView BasicStringView::findOr(const StringView substring, T* const fail) const { + /* Cache the getters to speed up debug builds */ + const std::size_t substringSize = substring.size(); + if(const char* const found = Implementation::stringFindString(_data, size(), substring._data, substringSize)) + return slice(const_cast(found), const_cast(found + substringSize)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findOr(const char character, T* const fail) const { + if(const char* const found = Implementation::stringFindCharacter(_data, size(), character)) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findLastOr(const StringView substring, T* const fail) const { + /* Cache the getters to speed up debug builds */ + const std::size_t substringSize = substring.size(); + if(const char* const found = Implementation::stringFindLastString(_data, size(), substring._data, substringSize)) + return slice(const_cast(found), const_cast(found + substringSize)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findLastOr(const char character, T* const fail) const { + if(const char* const found = Implementation::stringFindLastCharacter(_data, size(), character)) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline bool BasicStringView::contains(const StringView substring) const { + return Implementation::stringFindString(_data, size(), substring._data, substring.size()); +} + +template inline bool BasicStringView::contains(const char character) const { + return Implementation::stringFindCharacter(_data, size(), character); +} + +template inline BasicStringView BasicStringView::findAnyOr(const StringView characters, T* const fail) const { + if(const char* const found = Implementation::stringFindAny(_data, size(), characters._data, characters.size())) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline BasicStringView BasicStringView::findLastAnyOr(const StringView characters, T* const fail) const { + if(const char* const found = Implementation::stringFindLastAny(_data, size(), characters._data, characters.size())) + return slice(const_cast(found), const_cast(found + 1)); + + /* Using an internal assert-less constructor, the public constructor + asserts would be redundant. Since it's a zero-sized view, it doesn't + really make sense to try to preserve any flags. */ + return BasicStringView{fail, 0 /* empty, no flags */, nullptr}; +} + +template inline bool BasicStringView::containsAny(const StringView characters) const { + return Implementation::stringFindAny(_data, size(), characters._data, characters.size()); +} + }} #endif From 46e4ed0a5cc24a93ec1f5f5be9a22194bcb56bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 20 Jul 2022 18:02:38 +0200 Subject: [PATCH 48/79] Containers: split tests for string and character variant of find(). It doesn't make sense to have them together, given the implementations have vastly different complexity, and especially once SIMD appears. Also dropped a test for a single character being the whole string, since that doesn't really make any sense -- the string variant of it was there just to test that memstr() / strstr() isn't used, which would return true if both strings would have only a 0-terminated prefix. --- .../Containers/Test/StringViewTest.cpp | 231 +++++++++--------- 1 file changed, 114 insertions(+), 117 deletions(-) diff --git a/src/Corrade/Containers/Test/StringViewTest.cpp b/src/Corrade/Containers/Test/StringViewTest.cpp index 6c4413c69..d6eb5cd3e 100644 --- a/src/Corrade/Containers/Test/StringViewTest.cpp +++ b/src/Corrade/Containers/Test/StringViewTest.cpp @@ -169,16 +169,18 @@ struct StringViewTest: TestSuite::Tester { void trimmedNullView(); /* Tests also contains() */ - void find(); - void findMultipleOccurences(); - void findWholeString(); + void findString(); + void findStringMultipleOccurences(); + void findStringWhole(); + void findCharacter(); void findEmpty(); void findFlags(); void findOr(); - void findLast(); - void findLastMultipleOccurences(); - void findLastWholeString(); + void findLastString(); + void findLastStringMultipleOccurences(); + void findLastStringWhole(); + void findLastCharacter(); void findLastEmpty(); void findLastFlags(); void findLastOr(); @@ -275,16 +277,18 @@ StringViewTest::StringViewTest() { &StringViewTest::trimmedFlags, &StringViewTest::trimmedNullView, - &StringViewTest::find, - &StringViewTest::findMultipleOccurences, - &StringViewTest::findWholeString, + &StringViewTest::findString, + &StringViewTest::findStringMultipleOccurences, + &StringViewTest::findStringWhole, + &StringViewTest::findCharacter, &StringViewTest::findEmpty, &StringViewTest::findFlags, &StringViewTest::findOr, - &StringViewTest::findLast, - &StringViewTest::findLastMultipleOccurences, - &StringViewTest::findLastWholeString, + &StringViewTest::findLastString, + &StringViewTest::findLastStringMultipleOccurences, + &StringViewTest::findLastStringWhole, + &StringViewTest::findLastCharacter, &StringViewTest::findLastEmpty, &StringViewTest::findLastFlags, &StringViewTest::findLastOr, @@ -1484,7 +1488,7 @@ void StringViewTest::trimmedNullView() { CORRADE_VERIFY(!StringView{nullptr}.trimmed().data()); } -void StringViewTest::find() { +void StringViewTest::findString() { StringView a = "hello cursed\0world!"_s; /* Right at the start */ @@ -1542,9 +1546,56 @@ void StringViewTest::find() { StringView found = a.find("world!\0"_s); CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); + } +} - /* Single character at the start */ +void StringViewTest::findStringMultipleOccurences() { + StringView a = "so, hello hell hello! hello"_s; + + /* Multiple occurrences */ + { + CORRADE_VERIFY(a.contains("hello")); + + StringView found = a.find("hello"); + CORRADE_COMPARE(found, "hello"); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 4); + + /* First occurrence almost but not quite complete */ } { + CORRADE_VERIFY(a.contains("hello!")); + + StringView found = a.find("hello!"); + CORRADE_COMPARE(found, "hello!"); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 15); + } +} + +void StringViewTest::findStringWhole() { + StringView a = "hell"_s; + + /* Finding a substring that's the whole string should succeed */ + { + CORRADE_VERIFY(a.contains("hell")); + + StringView found = a.find("hell"); + CORRADE_COMPARE(found, "hell"); + CORRADE_COMPARE((static_cast(found.data())), a.data()); + + /* But a larger string should fail */ + } { + CORRADE_VERIFY(!a.contains("hello")); + + StringView found = a.find("hello"); + CORRADE_VERIFY(!found.data()); + CORRADE_VERIFY(found.isEmpty()); + } +} + +void StringViewTest::findCharacter() { + StringView a = "hello cursed\0world!"_s; + + /* Single character at the start */ + { CORRADE_VERIFY(a.contains('h')); StringView found = a.find('h'); @@ -1583,67 +1634,14 @@ void StringViewTest::find() { StringView found = a.exceptPrefix(15).find('\0'); CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); - } -} - -void StringViewTest::findMultipleOccurences() { - StringView a = "so, hello hell hello! hello"_s; /* Multiple occurrences */ - { - CORRADE_VERIFY(a.contains("hello")); - - StringView found = a.find("hello"); - CORRADE_COMPARE(found, "hello"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 4); - - /* First occurrence almost but not quite complete */ - } { - CORRADE_VERIFY(a.contains("hello!")); - - StringView found = a.find("hello!"); - CORRADE_COMPARE(found, "hello!"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 15); - - /* Multiple character occurrences */ } { CORRADE_VERIFY(a.contains('o')); StringView found = a.find('o'); CORRADE_COMPARE(found, "o"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 1); - } -} - -void StringViewTest::findWholeString() { - StringView a = "hell"_s; - - /* Finding a substring that's the whole string should succeed */ - { - CORRADE_VERIFY(a.contains("hell")); - - StringView found = a.find("hell"); - CORRADE_COMPARE(found, "hell"); - CORRADE_COMPARE((static_cast(found.data())), a.data()); - - /* But a larger string should fail */ - } { - CORRADE_VERIFY(!a.contains("hello")); - - StringView found = a.find("hello"); - CORRADE_VERIFY(!found.data()); - CORRADE_VERIFY(found.isEmpty()); - } - - StringView b = "h"_s; - - /* Finding a single character that's the whole string should succeed too */ - { - CORRADE_VERIFY(b.contains('h')); - - StringView found = b.find('h'); - CORRADE_COMPARE(found, "h"); - CORRADE_COMPARE((static_cast(found.data())), b.data()); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 4); } } @@ -1802,9 +1800,9 @@ void StringViewTest::findOr() { } } -void StringViewTest::findLast() { - /* Mostly similar to find(), except that it doesn't check contains() (which - is internally the same algorithm as find()) */ +void StringViewTest::findLastString() { + /* Mostly similar to findString(), except that it doesn't check contains() + (which is internally the same algorithm as find()) */ StringView a = "hello cursed\0world!"_s; @@ -1849,43 +1847,12 @@ void StringViewTest::findLast() { StringView found = a.findLast("world!\0"_s); CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); - - /* Single character at the end */ - } { - StringView found = a.findLast('!'); - CORRADE_COMPARE(found, "!"); - CORRADE_COMPARE(static_cast(found.data()), a.data() + 18); - - /* Single character in the middle */ - } { - StringView found = a.findLast('c'); - CORRADE_COMPARE(found, "c"); - CORRADE_COMPARE(static_cast(found.data()), a.data() + 6); - - /* Single character at the start */ - } { - StringView found = a.findLast('h'); - CORRADE_COMPARE(found, "h"); - CORRADE_COMPARE(static_cast(found.data()), a.data()); - - /* No such character found */ - } { - StringView found = a.findLast('a'); - CORRADE_VERIFY(!found.data()); - CORRADE_VERIFY(found.isEmpty()); - - /* Should not read the null terminator character either */ - } { - /* There's a \0 in the middle, skip that */ - StringView found = a.exceptPrefix(15).findLast('\0'); - CORRADE_VERIFY(!found.data()); - CORRADE_VERIFY(found.isEmpty()); } } -void StringViewTest::findLastMultipleOccurences() { - /* Mostly similar to findMultipleOccurences(), except that it doesn't check - contains() (which is internally the same algorithm as find()) */ +void StringViewTest::findLastStringMultipleOccurences() { + /* Mostly similar to findStringMultipleOccurences(), except that it doesn't + check contains() (which is internally the same algorithm as find()) */ StringView a = "so, hello hell hello! hello hell"_s; @@ -1900,17 +1867,11 @@ void StringViewTest::findLastMultipleOccurences() { StringView found = a.findLast("hello!"); CORRADE_COMPARE(found, "hello!"); CORRADE_COMPARE((static_cast(found.data())), a.data() + 15); - - /* Multiple character occurrences */ - } { - StringView found = a.findLast('o'); - CORRADE_COMPARE(found, "o"); - CORRADE_COMPARE((static_cast(found.data())), a.data() + 26); } } -void StringViewTest::findLastWholeString() { - /* Mostly similar to findWholeString(), except that it doesn't check +void StringViewTest::findLastStringWhole() { + /* Mostly similar to findStringWhole(), except that it doesn't check contains() (which is internally the same algorithm as find()) */ StringView a = "hell"_s; @@ -1927,14 +1888,50 @@ void StringViewTest::findLastWholeString() { CORRADE_VERIFY(!found.data()); CORRADE_VERIFY(found.isEmpty()); } +} - StringView b = "h"_s; +void StringViewTest::findLastCharacter() { + /* Mostly similar to findCharacter(), except that it doesn't check + contains() (which is internally the same algorithm as find()) */ + + StringView a = "hello cursed\0world!"_s; - /* Finding a single character that's the whole string should succeed too */ + /* Single character at the end */ { - StringView found = b.findLast('h'); + StringView found = a.findLast('!'); + CORRADE_COMPARE(found, "!"); + CORRADE_COMPARE(static_cast(found.data()), a.data() + 18); + + /* Single character in the middle */ + } { + StringView found = a.findLast('c'); + CORRADE_COMPARE(found, "c"); + CORRADE_COMPARE(static_cast(found.data()), a.data() + 6); + + /* Single character at the start */ + } { + StringView found = a.findLast('h'); CORRADE_COMPARE(found, "h"); - CORRADE_COMPARE((static_cast(found.data())), b.data()); + CORRADE_COMPARE(static_cast(found.data()), a.data()); + + /* No such character found */ + } { + StringView found = a.findLast('a'); + CORRADE_VERIFY(!found.data()); + CORRADE_VERIFY(found.isEmpty()); + + /* Should not read the null terminator character either */ + } { + /* There's a \0 in the middle, skip that */ + StringView found = a.exceptPrefix(15).findLast('\0'); + CORRADE_VERIFY(!found.data()); + CORRADE_VERIFY(found.isEmpty()); + + /* Multiple character occurrences */ + } { + StringView found = a.findLast('o'); + CORRADE_COMPARE(found, "o"); + CORRADE_COMPARE((static_cast(found.data())), a.data() + 14); } } From 272e8ef5d88888294aa615c556fc1ef8c0c292b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 20 Jul 2022 16:14:42 +0200 Subject: [PATCH 49/79] Containers: initial benchmark for a subset of StringView APIs. More APIs will be added as they get a potential for being SIMDified, and more variants depending on what SIMD corner cases appear. --- src/Corrade/Containers/Test/CMakeLists.txt | 15 + .../Test/StringTestFiles/lorem-ipsum.txt | 9 + .../Containers/Test/StringViewBenchmark.cpp | 414 ++++++++++++++++++ src/Corrade/Containers/Test/configure.h.cmake | 29 ++ 4 files changed, 467 insertions(+) create mode 100644 src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt create mode 100644 src/Corrade/Containers/Test/StringViewBenchmark.cpp create mode 100644 src/Corrade/Containers/Test/configure.h.cmake diff --git a/src/Corrade/Containers/Test/CMakeLists.txt b/src/Corrade/Containers/Test/CMakeLists.txt index 2488b669d..95481eee5 100644 --- a/src/Corrade/Containers/Test/CMakeLists.txt +++ b/src/Corrade/Containers/Test/CMakeLists.txt @@ -28,6 +28,15 @@ # property that would have to be set on each target separately. set(CMAKE_FOLDER "Corrade/Containers/Test") +if(CORRADE_TARGET_EMSCRIPTEN OR CORRADE_TARGET_ANDROID) + set(CONTAINERS_TEST_DIR ".") +else() + set(CONTAINERS_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +endif() + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake + ${CMAKE_CURRENT_BINARY_DIR}/configure.h) + corrade_add_test(ContainersAnyReferenceTest AnyReferenceTest.cpp) corrade_add_test(ContainersArrayTest ArrayTest.cpp) corrade_add_test(ContainersArrayTupleTest ArrayTupleTest.cpp LIBRARIES CorradeUtilityTestLib) @@ -72,6 +81,12 @@ corrade_add_test(ContainersStridedArrayViewStlTest StridedArrayViewStlTest.cpp) corrade_add_test(ContainersStringTest StringTest.cpp LIBRARIES CorradeUtilityTestLib) corrade_add_test(ContainersStringStlTest StringStlTest.cpp) corrade_add_test(ContainersStringViewTest StringViewTest.cpp LIBRARIES CorradeUtilityTestLib) + +corrade_add_test(ContainersStringViewBenchmark StringViewBenchmark.cpp + LIBRARIES CorradeUtilityTestLib + FILES StringTestFiles/lorem-ipsum.txt) +target_include_directories(ContainersStringViewBenchmark PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + corrade_add_test(ContainersTripleTest TripleTest.cpp) corrade_add_test(ContainersTripleStlTest TripleStlTest.cpp) diff --git a/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt b/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt new file mode 100644 index 000000000..91857a4a8 --- /dev/null +++ b/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt @@ -0,0 +1,9 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin suscipit pharetra elit, vel ornare dui sollicitudin vel. Sed bibendum erat ex, at gravida risus rhoncus a. Sed non placerat ipsum, vitae mattis metus. Duis libero nisi, ullamcorper euismod mauris sit amet, vulputate mollis leo. Vivamus rhoncus ante et nunc lobortis lobortis. Donec elementum felis id lorem volutpat, sed varius lectus blandit. Ut sit amet elit et diam eleifend tincidunt. Vivamus rutrum consequat euismod. Nullam pretium felis eget arcu tincidunt congue. Fusce at suscipit tellus. Sed efficitur leo at ligula tempus, eget dignissim ex lobortis. + +Maecenas sit amet ligula metus. Phasellus non pretium felis. Suspendisse at metus in magna viverra sollicitudin. Etiam felis lectus, facilisis ac malesuada non, laoreet sed arcu. Nulla congue nulla justo, vitae elementum massa mollis eu. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Aliquam commodo sapien enim, eleifend tempor leo feugiat a. Integer tincidunt, ligula at vulputate posuere, nulla diam sollicitudin augue, vitae suscipit lectus lorem nec risus. Duis ex metus, elementum sed libero scelerisque, faucibus sollicitudin elit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras ac ultrices tortor, in fringilla augue. In magna nibh, molestie id lacinia ut, hendrerit sit amet lacus. + +Pellentesque ac risus sed ligula condimentum consectetur. Ut eu dolor faucibus justo consectetur imperdiet nec id risus. Etiam sodales blandit est, et consectetur ex rutrum ut. Nullam at dignissim lorem. Cras non neque a sem malesuada tincidunt et nec elit. Vivamus dignissim elit eu auctor eleifend. Donec convallis quam eu aliquet gravida. Curabitur porttitor ipsum lectus, quis varius ex vulputate ut. + +Morbi a mattis nunc. Mauris suscipit orci eget nibh tincidunt aliquet. Praesent maximus ullamcorper ligula sed pellentesque. In hac habitasse platea dictumst. Suspendisse at purus nisl. Curabitur quis odio est. Ut tempor nunc quis magna euismod, vitae interdum massa faucibus. Ut in porta mauris, rhoncus hendrerit mi. Quisque orci lacus, malesuada vel augue eget, ultricies pretium nisi. Suspendisse metus ex, tristique ac facilisis at, bibendum vel eros. Integer eleifend nibh a diam malesuada, in lacinia leo condimentum. Duis dignissim rutrum odio, nec feugiat sapien elementum nec. Donec quis augue elit. Cras laoreet, sapien sed rhoncus dapibus, ligula mauris placerat enim, sed mattis sem ipsum sit amet mi. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed imperdiet purus urna, volutpat dictum ante interdum in. + +Mauris a massa pharetra, gravida ligula a, fringilla orci. Praesent tempor pretium pretium. Mauris tincidunt pellentesque maximus. Quisque eget mauris diam. In nec porttitor felis, et finibus mauris. Donec quis quam risus. Vivamus blandit ornare massa, in vulputate purus tincidunt a. Nam at erat non lorem iaculis faucibus. Nulla eu magna odio. Mauris dignissim tempor velit, ut pretium eros dignissim id. Duis fermentum viverra lectus ut dapibus. Nullam id turpis efficitur velit iaculis varius vitae vel nunc. Morbi urna velit, consequat mollis dui nec, vestibulum dictum tellus. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque in pulvinar lacus, et vulputate turpis. diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp new file mode 100644 index 000000000..b1915915a --- /dev/null +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -0,0 +1,414 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include +#include + +#include "Corrade/Containers/Optional.h" +#include "Corrade/Containers/StringView.h" +#include "Corrade/Containers/StringStl.h" +#include "Corrade/TestSuite/Tester.h" +#include "Corrade/Utility/Path.h" + +#include "configure.h" + +namespace Corrade { namespace Containers { namespace Test { namespace { + +struct StringViewBenchmark: TestSuite::Tester { + explicit StringViewBenchmark(); + + /* The "Common" variants test rather the call / preamble / postamble + overhead, while the "Rare" variants test the actual vectorized + implementation perf */ + + void findCharacterCommon(); + void findCharacterCommonNaive(); + void findCharacterCommonMemchr(); + void findCharacterCommonStlString(); + + void findCharacterRare(); + void findCharacterRareNaive(); + void findCharacterRareMemchr(); + void findCharacterRareStlString(); + + void findLastCharacterCommon(); + void findLastCharacterCommonNaive(); + void findLastCharacterCommonMemrchr(); + void findLastCharacterCommonStlString(); + + void findLastCharacterRare(); + void findLastCharacterRareNaive(); + void findLastCharacterRareMemrchr(); + void findLastCharacterRareStlString(); +}; + +using namespace Containers::Literals; + +StringViewBenchmark::StringViewBenchmark() { + addBenchmarks({&StringViewBenchmark::findCharacterCommon, + &StringViewBenchmark::findCharacterCommonNaive, + &StringViewBenchmark::findCharacterCommonMemchr, + &StringViewBenchmark::findCharacterCommonStlString, + + &StringViewBenchmark::findCharacterRare, + &StringViewBenchmark::findCharacterRareNaive, + &StringViewBenchmark::findCharacterRareMemchr, + &StringViewBenchmark::findCharacterRareStlString, + + &StringViewBenchmark::findLastCharacterCommon, + &StringViewBenchmark::findLastCharacterCommonNaive, + &StringViewBenchmark::findLastCharacterCommonMemrchr, + &StringViewBenchmark::findLastCharacterCommonStlString, + + &StringViewBenchmark::findLastCharacterRare, + &StringViewBenchmark::findLastCharacterRareNaive, + &StringViewBenchmark::findLastCharacterRareMemrchr, + &StringViewBenchmark::findLastCharacterRareStlString}, 100); +} + +constexpr std::size_t CommonCharacterCount = 496; +constexpr std::size_t RareCharacterCount = 90; +constexpr std::size_t CharacterRepeats = 100; + +void StringViewBenchmark::findCharacterCommon() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.find(' ')) { + ++count; + a = a.suffix(found.end()); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + for(;;) { + const char* found = nullptr; + for(const char* i = a; i != text->end(); ++i) { + if(*i == ' ') { + found = i; + break; + } + } + if(!found) break; + + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonMemchr() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + while(const char* found = static_cast(std::memchr(a, ' ', text->end() - a))) { + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t pos = 0; + std::size_t found; + while((found = a.find(' ', pos)) != std::string::npos) { + ++count; + pos = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRare() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.find('\n')) { + ++count; + a = a.suffix(found.end()); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRareNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + for(;;) { + const char* found = nullptr; + for(const char* i = a; i != text->end(); ++i) { + if(*i == '\n') { + found = i; + break; + } + } + if(!found) break; + + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRareMemchr() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + while(const char* found = static_cast(std::memchr(a, '\n', text->end() - a))) { + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterRareStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t pos = 0; + std::size_t found; + while((found = a.find('\n', pos)) != std::string::npos) { + ++count; + pos = found + 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommon() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.findLast(' ')) { + ++count; + a = a.prefix(found.begin()); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommonNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + for(;;) { + const char* found = nullptr; + for(const char* i = text->begin() + end; i != text->begin(); --i) { + if(*(i - 1) == ' ') { + found = i - 1; + break; + } + } + if(!found) break; + + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommonMemrchr() { + #if !defined(__GLIBC__) && !defined(__BIONIC__) && !defined(CORRADE_TARGET_EMSCRIPTEN) + CORRADE_SKIP("memrchr() not available"); + #else + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + while(const char* found = static_cast(memrchr(text->begin(), ' ', end))) { + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); + #endif +} + +void StringViewBenchmark::findLastCharacterCommonStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + std::size_t found; + while((found = a.rfind(' ', end)) != std::string::npos) { + ++count; + end = found - 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterRare() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.findLast('\n')) { + ++count; + a = a.prefix(found.begin()); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterRareNaive() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + for(;;) { + const char* found = nullptr; + for(const char* i = text->begin() + end; i != text->begin(); --i) { + if(*(i - 1) == '\n') { + found = i - 1; + break; + } + } + if(!found) break; + + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterRareMemrchr() { + #if !defined(__GLIBC__) && !defined(__BIONIC__) && !defined(CORRADE_TARGET_EMSCRIPTEN) + CORRADE_SKIP("memrchr() not available"); + #else + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + while(const char* found = static_cast(memrchr(text->begin(), '\n', end))) { + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); + #endif +} + +void StringViewBenchmark::findLastCharacterRareStlString() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + *text = *text*10; + + std::size_t count = 0; + std::string a = *text; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + std::size_t found; + while((found = a.rfind('\n', end)) != std::string::npos) { + ++count; + end = found - 1; + } + } + + CORRADE_COMPARE(count, RareCharacterCount*CharacterRepeats); +} + +}}}} + +CORRADE_TEST_MAIN(Corrade::Containers::Test::StringViewBenchmark) diff --git a/src/Corrade/Containers/Test/configure.h.cmake b/src/Corrade/Containers/Test/configure.h.cmake new file mode 100644 index 000000000..c5bb625fb --- /dev/null +++ b/src/Corrade/Containers/Test/configure.h.cmake @@ -0,0 +1,29 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#define CONTAINERS_TEST_DIR "${CONTAINERS_TEST_DIR}" + +// kate: hl c++ From ab543ab993b8be239e8748e69f8bd7c43991217e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 28 Jul 2022 20:36:49 +0200 Subject: [PATCH 50/79] Containers: benchmark also String::find[Last]() with small sizes. To test the scalar fallback for less than 16 bytes. Had to patch the test data to have spaces also inside the paragraph breaks, as it's a too large range without a space otherwise. --- .../Test/StringTestFiles/lorem-ipsum.txt | 8 +- .../Containers/Test/StringViewBenchmark.cpp | 89 ++++++++++++++++++- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt b/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt index 91857a4a8..184415cd1 100644 --- a/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt +++ b/src/Corrade/Containers/Test/StringTestFiles/lorem-ipsum.txt @@ -1,9 +1,9 @@ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin suscipit pharetra elit, vel ornare dui sollicitudin vel. Sed bibendum erat ex, at gravida risus rhoncus a. Sed non placerat ipsum, vitae mattis metus. Duis libero nisi, ullamcorper euismod mauris sit amet, vulputate mollis leo. Vivamus rhoncus ante et nunc lobortis lobortis. Donec elementum felis id lorem volutpat, sed varius lectus blandit. Ut sit amet elit et diam eleifend tincidunt. Vivamus rutrum consequat euismod. Nullam pretium felis eget arcu tincidunt congue. Fusce at suscipit tellus. Sed efficitur leo at ligula tempus, eget dignissim ex lobortis. -Maecenas sit amet ligula metus. Phasellus non pretium felis. Suspendisse at metus in magna viverra sollicitudin. Etiam felis lectus, facilisis ac malesuada non, laoreet sed arcu. Nulla congue nulla justo, vitae elementum massa mollis eu. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Aliquam commodo sapien enim, eleifend tempor leo feugiat a. Integer tincidunt, ligula at vulputate posuere, nulla diam sollicitudin augue, vitae suscipit lectus lorem nec risus. Duis ex metus, elementum sed libero scelerisque, faucibus sollicitudin elit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras ac ultrices tortor, in fringilla augue. In magna nibh, molestie id lacinia ut, hendrerit sit amet lacus. + Maecenas sit amet ligula metus. Phasellus non pretium felis. Suspendisse at metus in magna viverra sollicitudin. Etiam felis lectus, facilisis ac malesuada non, laoreet sed arcu. Nulla congue nulla justo, vitae elementum massa mollis eu. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Aliquam commodo sapien enim, eleifend tempor leo feugiat a. Integer tincidunt, ligula at vulputate posuere, nulla diam sollicitudin augue, vitae suscipit lectus lorem nec risus. Duis ex metus, elementum sed libero scelerisque, faucibus sollicitudin elit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras ac ultrices tortor, in fringilla augue. In magna nibh, molestie id lacinia ut, hendrerit sit amet lacus. -Pellentesque ac risus sed ligula condimentum consectetur. Ut eu dolor faucibus justo consectetur imperdiet nec id risus. Etiam sodales blandit est, et consectetur ex rutrum ut. Nullam at dignissim lorem. Cras non neque a sem malesuada tincidunt et nec elit. Vivamus dignissim elit eu auctor eleifend. Donec convallis quam eu aliquet gravida. Curabitur porttitor ipsum lectus, quis varius ex vulputate ut. + Pellentesque ac risus sed ligula condimentum consectetur. Ut eu dolor faucibus justo consectetur imperdiet nec id risus. Etiam sodales blandit est, et consectetur ex rutrum ut. Nullam at dignissim lorem. Cras non neque a sem malesuada tincidunt et nec elit. Vivamus dignissim elit eu auctor eleifend. Donec convallis quam eu aliquet gravida. Curabitur porttitor ipsum lectus, quis varius ex vulputate ut. -Morbi a mattis nunc. Mauris suscipit orci eget nibh tincidunt aliquet. Praesent maximus ullamcorper ligula sed pellentesque. In hac habitasse platea dictumst. Suspendisse at purus nisl. Curabitur quis odio est. Ut tempor nunc quis magna euismod, vitae interdum massa faucibus. Ut in porta mauris, rhoncus hendrerit mi. Quisque orci lacus, malesuada vel augue eget, ultricies pretium nisi. Suspendisse metus ex, tristique ac facilisis at, bibendum vel eros. Integer eleifend nibh a diam malesuada, in lacinia leo condimentum. Duis dignissim rutrum odio, nec feugiat sapien elementum nec. Donec quis augue elit. Cras laoreet, sapien sed rhoncus dapibus, ligula mauris placerat enim, sed mattis sem ipsum sit amet mi. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed imperdiet purus urna, volutpat dictum ante interdum in. + Morbi a mattis nunc. Mauris suscipit orci eget nibh tincidunt aliquet. Praesent maximus ullamcorper ligula sed pellentesque. In hac habitasse platea dictumst. Suspendisse at purus nisl. Curabitur quis odio est. Ut tempor nunc quis magna euismod, vitae interdum massa faucibus. Ut in porta mauris, rhoncus hendrerit mi. Quisque orci lacus, malesuada vel augue eget, ultricies pretium nisi. Suspendisse metus ex, tristique ac facilisis at, bibendum vel eros. Integer eleifend nibh a diam malesuada, in lacinia leo condimentum. Duis dignissim rutrum odio, nec feugiat sapien elementum nec. Donec quis augue elit. Cras laoreet, sapien sed rhoncus dapibus, ligula mauris placerat enim, sed mattis sem ipsum sit amet mi. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed imperdiet purus urna, volutpat dictum ante interdum in. -Mauris a massa pharetra, gravida ligula a, fringilla orci. Praesent tempor pretium pretium. Mauris tincidunt pellentesque maximus. Quisque eget mauris diam. In nec porttitor felis, et finibus mauris. Donec quis quam risus. Vivamus blandit ornare massa, in vulputate purus tincidunt a. Nam at erat non lorem iaculis faucibus. Nulla eu magna odio. Mauris dignissim tempor velit, ut pretium eros dignissim id. Duis fermentum viverra lectus ut dapibus. Nullam id turpis efficitur velit iaculis varius vitae vel nunc. Morbi urna velit, consequat mollis dui nec, vestibulum dictum tellus. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque in pulvinar lacus, et vulputate turpis. + Mauris a massa pharetra, gravida ligula a, fringilla orci. Praesent tempor pretium pretium. Mauris tincidunt pellentesque maximus. Quisque eget mauris diam. In nec porttitor felis, et finibus mauris. Donec quis quam risus. Vivamus blandit ornare massa, in vulputate purus tincidunt a. Nam at erat non lorem iaculis faucibus. Nulla eu magna odio. Mauris dignissim tempor velit, ut pretium eros dignissim id. Duis fermentum viverra lectus ut dapibus. Nullam id turpis efficitur velit iaculis varius vitae vel nunc. Morbi urna velit, consequat mollis dui nec, vestibulum dictum tellus. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque in pulvinar lacus, et vulputate turpis. diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp index b1915915a..838654be5 100644 --- a/src/Corrade/Containers/Test/StringViewBenchmark.cpp +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -31,6 +31,7 @@ #include "Corrade/Containers/StringView.h" #include "Corrade/Containers/StringStl.h" #include "Corrade/TestSuite/Tester.h" +#include "Corrade/Utility/Math.h" #include "Corrade/Utility/Path.h" #include "configure.h" @@ -49,6 +50,11 @@ struct StringViewBenchmark: TestSuite::Tester { void findCharacterCommonMemchr(); void findCharacterCommonStlString(); + void findCharacterCommonSmall(); + void findCharacterCommonSmallMemchr(); + /* No std::string variant as the overhead from slicing would make this + useless (and no, find() has no end position) */ + void findCharacterRare(); void findCharacterRareNaive(); void findCharacterRareMemchr(); @@ -59,6 +65,11 @@ struct StringViewBenchmark: TestSuite::Tester { void findLastCharacterCommonMemrchr(); void findLastCharacterCommonStlString(); + void findLastCharacterCommonSmall(); + void findLastCharacterCommonSmallMemrchr(); + /* No std::string variant as the overhead from slicing would make this + useless (and no, rfind() has no end position) */ + void findLastCharacterRare(); void findLastCharacterRareNaive(); void findLastCharacterRareMemrchr(); @@ -73,6 +84,9 @@ StringViewBenchmark::StringViewBenchmark() { &StringViewBenchmark::findCharacterCommonMemchr, &StringViewBenchmark::findCharacterCommonStlString, + &StringViewBenchmark::findCharacterCommonSmall, + &StringViewBenchmark::findCharacterCommonSmallMemchr, + &StringViewBenchmark::findCharacterRare, &StringViewBenchmark::findCharacterRareNaive, &StringViewBenchmark::findCharacterRareMemchr, @@ -83,13 +97,16 @@ StringViewBenchmark::StringViewBenchmark() { &StringViewBenchmark::findLastCharacterCommonMemrchr, &StringViewBenchmark::findLastCharacterCommonStlString, + &StringViewBenchmark::findLastCharacterCommonSmall, + &StringViewBenchmark::findLastCharacterCommonSmallMemrchr, + &StringViewBenchmark::findLastCharacterRare, &StringViewBenchmark::findLastCharacterRareNaive, &StringViewBenchmark::findLastCharacterRareMemrchr, &StringViewBenchmark::findLastCharacterRareStlString}, 100); } -constexpr std::size_t CommonCharacterCount = 496; +constexpr std::size_t CommonCharacterCount = 500; constexpr std::size_t RareCharacterCount = 90; constexpr std::size_t CharacterRepeats = 100; @@ -168,6 +185,39 @@ void StringViewBenchmark::findCharacterCommonStlString() { CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); } +void StringViewBenchmark::findCharacterCommonSmall() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + while(StringView found = a.prefix(Utility::min(std::size_t{15}, a.size())).find(' ')) { + ++count; + a = a.suffix(found.end()); + } + } + + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findCharacterCommonSmallMemchr() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + const char* a = text->data(); + while(const char* found = static_cast(std::memchr(a, ' ', Utility::min(std::ptrdiff_t{15}, text->end() - a)))) { + ++count; + a = found + 1; + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + void StringViewBenchmark::findCharacterRare() { Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); CORRADE_VERIFY(text); @@ -326,6 +376,43 @@ void StringViewBenchmark::findLastCharacterCommonStlString() { CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); } +void StringViewBenchmark::findLastCharacterCommonSmall() { + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + StringView a = *text; + /** @todo use suffix() once it takes suffix size */ + while(StringView found = a.exceptPrefix(Utility::max(std::ptrdiff_t{0}, std::ptrdiff_t(a.size()) - 15)).findLast(' ')) { + ++count; + a = a.prefix(found.begin()); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); +} + +void StringViewBenchmark::findLastCharacterCommonSmallMemrchr() { + #if !defined(__GLIBC__) && !defined(__BIONIC__) && !defined(CORRADE_TARGET_EMSCRIPTEN) + CORRADE_SKIP("memrchr() not available"); + #else + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); + CORRADE_VERIFY(text); + + std::size_t count = 0; + CORRADE_BENCHMARK(CharacterRepeats) { + std::size_t end = text->size(); + while(const char* found = static_cast(memrchr(text->begin() + Utility::max(std::ptrdiff_t{0}, std::ptrdiff_t(end) - 15), ' ', end - Utility::max(std::ptrdiff_t{0}, std::ptrdiff_t(end) - 15)))) { + ++count; + end = found - text->begin(); + } + } + + CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); + #endif +} + void StringViewBenchmark::findLastCharacterRare() { Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); CORRADE_VERIFY(text); From 6f84587b4c9e40d76d65d9ea063f2c36e748ce75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 22 Jul 2022 18:53:02 +0200 Subject: [PATCH 51/79] Add a CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH option. The tests will be implicitly configured to use function pointers for CPU dispatch, but since it adds significant overhead on Emscripten, there also needs to be a way to disable this. --- CMakeLists.txt | 5 +++++ doc/building-corrade.dox | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c2f55c8d5..5f1050436 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -256,6 +256,11 @@ cmake_dependent_option(CORRADE_CPU_USE_IFUNC "Allow using GNU IFUNC for runtime # negative perf consequences, enable it by default only on platforms that have # IFUNC, and thus can avoid the function pointer indirection. option(CORRADE_BUILD_CPU_RUNTIME_DISPATCH "Build with runtime dispatch for CPU-dependent functionality" ${_CORRADE_CPU_CAN_USE_IFUNC}) +# Enabled by default and independent of CORRADE_BUILD_CPU_RUNTIME_DISPATCH as +# correctness is important. It can however negatively affect performance +# compared to IFUNC or compile-time dispatch, so make it possible to disable +# this for benchmarks. +cmake_dependent_option(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH "Force pointer-based dispatch for unit tests to verify all variants of CPU-dependent functionality" ON "CORRADE_BUILD_TESTS" OFF) # Backwards compatibility for unprefixed CMake options. If the user isn't # explicitly using prefixed options in the first run already, accept the diff --git a/doc/building-corrade.dox b/doc/building-corrade.dox index 53457bd17..1aa463e52 100644 --- a/doc/building-corrade.dox +++ b/doc/building-corrade.dox @@ -641,11 +641,21 @@ compatibility if `CORRADE_BUILD_DEPRECATED` isn't disabled. @subsection building-corrade-tests Building and running tests -If you want to build also the tests (which are not built by default), enable -`CORRADE_BUILD_TESTS` in CMake. The tests use Corrade's own -@ref Corrade::TestSuite "TestSuite" framework and can be run either manually -(the binaries are located in `Test/` subdirectories in the build directory) or -using +Building of tests is controlled by the following options: + +- `CORRADE_BUILD_TESTS` --- Builds unit tests. Disabled by default. +- `CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH` --- Force unit tests to be + built with function-pointer-based dispatch for CPU-dependent functionality, + independently of the `CORRADE_BUILD_CPU_RUNTIME_DISPATCH` and + `CORRADE_CPU_USE_IFUNC` options. This makes it possible for the tests to + verify all variants instead of just one, and is enabled by default. The + overhead from function pointers may however be significant on certain + platforms, skewing benchmark results. Disable it to make the tests use the + same dispatch method as the rest of the library. + +The tests use Corrade's own @relativeref{Corrade,TestSuite} framework and can +be run either manually (the binaries are located in `Test/` subdirectories in +the build directory) or using @code{.sh} ctest --output-on-failure From 6c8467ecdbc90eff3d42e60d5f854c18b3632b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 20:00:09 +0200 Subject: [PATCH 52/79] Cpu: document more GCC lambda attribute bugs. What a minefield! And this is C++11 functionality, the ancient version that only obsolete cavemen like me use! Can't even imagine what hell must be developing some C++17 stuff. Ugh!! --- src/Corrade/Cpu.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index e703a518c..d4c009f8e 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -304,7 +304,7 @@ actual arguments: -@m_class{m-block m-info } +@m_class{m-block m-danger } @par Instruction enabling macros An important difference with the @ref CORRADE_ENABLE_SSE2 "CORRADE_ENABLE_*" @@ -313,8 +313,9 @@ actual arguments: from the wrapper function to the nested lambda. To make matters worse, older versions of Clang suffer from the inverse problem and ignore lambda attributes, so you have to specify them on both the lambda and the wrapper - function. The above AVX variant would look like this with relevant macros - added: + function. GCC 9.1 to 9.3 also has a [bug where it can't parse lambda attributes](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333), + use version 8, 9.4 or 10.3 instead. The above AVX variant would look like + this with relevant macros added: @par @snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-target-attributes From 6615e449b793fa0f7b1d1db5a1318d916627bac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 18:50:29 +0200 Subject: [PATCH 53/79] package/ci: install GCC 9.4 on the Android build. 9.3 has https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333, which breaks the CPU dispatch. Since 9.4 is readily available (and just not a default) I don't want to bother working around this. --- package/ci/circleci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/package/ci/circleci.yml b/package/ci/circleci.yml index 41aec4405..37511ab58 100644 --- a/package/ci/circleci.yml +++ b/package/ci/circleci.yml @@ -89,10 +89,14 @@ commands: background: true - run: name: Update apt and install base packages - # Compared to Docker images this needs sudo + # Compared to Docker images this needs sudo. Default gcc is 9.3, which + # has https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 and breaks our + # CPU dispatch, 9.4 is for some reason a separate package. command: | sudo apt update - sudo apt install -y ninja-build + sudo apt install -y ninja-build g++-9 + echo 'export CC=gcc-9' >> $BASH_ENV + echo 'export CXX=g++-9' >> $BASH_ENV install-gcc-4_8: steps: From 98c4744425ff3c49fbf130ea51c10e9d7bd58334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 19:39:54 +0200 Subject: [PATCH 54/79] CMake: avoid linking to both CorradeUtilityTestLib and CorradeUtility. The upcoming function pointer dispatch would mean there's two conflicting exported global function pointers for each dispatched function, and that makes ASan extremely upset about an ODR violation. This weird dependency was caused by tests being implicitly linked to CorradeTestSuite, which depends on CorradeUtility. Meaning, if a test linked to CorradeUtilityTestLib, CorradeUtility would be dragged in as well. Instead, there's now CorradeTestSuiteTestLib, which sole purpose is to link to CorradeUtilityTestLib instead of CorradeUtililty, and this library is now linked to instead of CorradeUtilityTestLib in all affected tests. What's also kinda weird and wrong is that CorradeUtilityTestLib linked to CorradeUtility instead of just reusing CorradeUtilityObjects. Traced that back to 5a9343a71a60892d0db2c74ce388af0805207753 from 2013, where this attempted to fix a build issue with MSVC. Which would break as soon as Debug.cpp would be a part of CorradeUtility_GracefulAssert_SRCS instead of the object library. The fix I have now is better in every aspect. --- modules/UseCorrade.cmake | 14 ++++++- src/Corrade/Containers/Test/CMakeLists.txt | 16 +++++--- src/Corrade/TestSuite/CMakeLists.txt | 45 +++++++++++++++++----- src/Corrade/TestSuite/visibility.h | 2 +- src/Corrade/Utility/CMakeLists.txt | 16 ++++++-- src/Corrade/Utility/Test/CMakeLists.txt | 24 +++++++----- 6 files changed, 87 insertions(+), 30 deletions(-) diff --git a/modules/UseCorrade.cmake b/modules/UseCorrade.cmake index af1726e68..7093ede76 100644 --- a/modules/UseCorrade.cmake +++ b/modules/UseCorrade.cmake @@ -346,6 +346,10 @@ function(corrade_add_test test_name) set(_corrade_file_pair_match "^(.+)@([^@]+)$") set(_corrade_file_pair_replace "\\1;\\2") + # TestSuite library to link to. Gets reset below if the tests already link + # to their own variant. + set(testsuite_library Corrade::TestSuite) + # Get DLL and path lists foreach(arg ${ARGN}) if(arg STREQUAL LIBRARIES) @@ -359,6 +363,12 @@ function(corrade_add_test test_name) set(_DOING_ARGUMENTS ON) else() if(_DOING_LIBRARIES) + # If Corrade's own tests link their own variant of TestSuite, + # don't link the implicit one as well, as otherwise we'd end + # up with duplicated symbols. + if(arg STREQUAL CorradeTestSuiteTestLib) + set(testsuite_library ) + endif() list(APPEND libraries ${arg}) elseif(_DOING_FILES) # If the file is already a pair of file and destination, just @@ -404,7 +414,7 @@ function(corrade_add_test test_name) add_library(${test_name} SHARED ${sources}) set_target_properties(${test_name} PROPERTIES FRAMEWORK TRUE) # This is never Windows, so no need to bother with Corrade::Main - target_link_libraries(${test_name} PRIVATE ${libraries} Corrade::TestSuite) + target_link_libraries(${test_name} PRIVATE ${libraries} ${testsuite_library}) set(test_runner_file ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.mm) configure_file(${CORRADE_TESTSUITE_XCTEST_RUNNER} @@ -431,7 +441,7 @@ function(corrade_add_test test_name) endif() else() add_executable(${test_name} ${sources}) - target_link_libraries(${test_name} PRIVATE ${libraries} Corrade::TestSuite Corrade::Main) + target_link_libraries(${test_name} PRIVATE ${libraries} ${testsuite_library} Corrade::Main) # Run tests using Node.js on Emscripten if(CORRADE_TARGET_EMSCRIPTEN) diff --git a/src/Corrade/Containers/Test/CMakeLists.txt b/src/Corrade/Containers/Test/CMakeLists.txt index 95481eee5..07eded83d 100644 --- a/src/Corrade/Containers/Test/CMakeLists.txt +++ b/src/Corrade/Containers/Test/CMakeLists.txt @@ -37,13 +37,19 @@ endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/configure.h) +# In all following corrade_add_test() macros, if a test wants to use +# CorradeUtilityTestLib, it has to link to CorradeTestSuiteTestLib instead. +# Otherwise the implicitly linked CorradeTestSuite would drag in CorradeUtility +# in addition to CorradeUtilityTestLib, leading to ODR violations and making +# ASan builds fail. + corrade_add_test(ContainersAnyReferenceTest AnyReferenceTest.cpp) corrade_add_test(ContainersArrayTest ArrayTest.cpp) -corrade_add_test(ContainersArrayTupleTest ArrayTupleTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersArrayTupleTest ArrayTupleTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(ContainersArrayViewTest ArrayViewTest.cpp) corrade_add_test(ContainersArrayViewStlTest ArrayViewStlTest.cpp) corrade_add_test(ContainersBigEnumSetTest BigEnumSetTest.cpp) -corrade_add_test(ContainersBitArrayTest BitArrayTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersBitArrayTest BitArrayTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(ContainersBitArrayViewTest BitArrayViewTest.cpp) corrade_add_test(ContainersEnumSetTest EnumSetTest.cpp) @@ -78,12 +84,12 @@ corrade_add_test(ContainersStaticArrayViewTest StaticArrayViewTest.cpp) corrade_add_test(ContainersStaticArrayViewStlTest StaticArrayViewStlTest.cpp) corrade_add_test(ContainersStridedArrayViewTest StridedArrayViewTest.cpp) corrade_add_test(ContainersStridedArrayViewStlTest StridedArrayViewStlTest.cpp) -corrade_add_test(ContainersStringTest StringTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersStringTest StringTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(ContainersStringStlTest StringStlTest.cpp) -corrade_add_test(ContainersStringViewTest StringViewTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(ContainersStringViewTest StringViewTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(ContainersStringViewBenchmark StringViewBenchmark.cpp - LIBRARIES CorradeUtilityTestLib + LIBRARIES CorradeTestSuiteTestLib FILES StringTestFiles/lorem-ipsum.txt) target_include_directories(ContainersStringViewBenchmark PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/src/Corrade/TestSuite/CMakeLists.txt b/src/Corrade/TestSuite/CMakeLists.txt index e6670e0dd..d300ac6ed 100644 --- a/src/Corrade/TestSuite/CMakeLists.txt +++ b/src/Corrade/TestSuite/CMakeLists.txt @@ -48,19 +48,18 @@ set(CorradeTestSuite_PRIVATE_HEADERS Implementation/BenchmarkCounters.h Implementation/BenchmarkStats.h) -# TestSuite library -add_library(CorradeTestSuite ${SHARED_OR_STATIC} +# Objects shared between main and test library +add_library(CorradeTestSuiteObjects OBJECT ${CorradeTestSuite_SRCS} ${CorradeTestSuite_HEADERS} ${CorradeTestSuite_PRIVATE_HEADERS}) -set_target_properties(CorradeTestSuite PROPERTIES - DEBUG_POSTFIX "-d") +target_include_directories(CorradeTestSuiteObjects PUBLIC $) if(NOT CORRADE_BUILD_STATIC) - set_target_properties(CorradeTestSuite PROPERTIES VERSION ${CORRADE_LIBRARY_VERSION} SOVERSION ${CORRADE_LIBRARY_SOVERSION}) -elseif(CORRADE_BUILD_STATIC_PIC) - set_target_properties(CorradeTestSuite PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(CorradeTestSuiteObjects PRIVATE "-DCorradeTestSuiteObjects_EXPORTS") +endif() +if(NOT CORRADE_BUILD_STATIC OR CORRADE_BUILD_STATIC_PIC) + set_target_properties(CorradeTestSuiteObjects PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() -target_link_libraries(CorradeTestSuite CorradeUtility) if(CORRADE_TARGET_EMSCRIPTEN) # Since (probably) 1.38.36, the compiler needs to have exceptions enabled # as well (not just the linker), and directly for the TestSuite library. @@ -69,9 +68,21 @@ if(CORRADE_TARGET_EMSCRIPTEN) # so doing it the old way --- and instead of the options being transitively # passed to the target, for the test target it's done explicitly inside # UseCorrade.cmake. Wasn't needed with 1.38.32. - set_property(TARGET CorradeTestSuite APPEND_STRING PROPERTY COMPILE_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") + set_property(TARGET CorradeTestSuiteObjects APPEND_STRING PROPERTY COMPILE_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") endif() +# Main TestSuite library +add_library(CorradeTestSuite ${SHARED_OR_STATIC} + $) +set_target_properties(CorradeTestSuite PROPERTIES + DEBUG_POSTFIX "-d") +if(NOT CORRADE_BUILD_STATIC) + set_target_properties(CorradeTestSuite PROPERTIES VERSION ${CORRADE_LIBRARY_VERSION} SOVERSION ${CORRADE_LIBRARY_SOVERSION}) +elseif(CORRADE_BUILD_STATIC_PIC) + set_target_properties(CorradeTestSuite PROPERTIES POSITION_INDEPENDENT_CODE ON) +endif() +target_link_libraries(CorradeTestSuite PUBLIC CorradeUtility) + install(TARGETS CorradeTestSuite RUNTIME DESTINATION ${CORRADE_BINARY_INSTALL_DIR} LIBRARY DESTINATION ${CORRADE_LIBRARY_INSTALL_DIR} @@ -88,6 +99,22 @@ endif() add_subdirectory(Compare) if(CORRADE_BUILD_TESTS) + # Library that links against CorradeUtilityTestLib instead of + # CorradeUtility for testing. No other difference. Not actually used by any + # TestSuite tests, but a special case needed only by the Utility library + # -- since TestSuite depends on Utility, Utility tests that would link to + # CorradeTestSuite would get CorradeUtility in addition to + # CorradeUtilityTestLib, leading to ODR violations and subsequently ASan + # complaints. + add_library(CorradeTestSuiteTestLib ${SHARED_OR_STATIC} + $) + target_include_directories(CorradeTestSuiteTestLib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + set_target_properties(CorradeTestSuiteTestLib PROPERTIES DEBUG_POSTFIX "-d") + if(CORRADE_BUILD_STATIC_PIC) + set_target_properties(CorradeTestSuiteTestLib PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() + target_link_libraries(CorradeTestSuiteTestLib PUBLIC CorradeUtilityTestLib) + add_subdirectory(Test) endif() diff --git a/src/Corrade/TestSuite/visibility.h b/src/Corrade/TestSuite/visibility.h index 4c0d90e0a..d7fd28d87 100644 --- a/src/Corrade/TestSuite/visibility.h +++ b/src/Corrade/TestSuite/visibility.h @@ -31,7 +31,7 @@ #ifndef DOXYGEN_GENERATING_OUTPUT #ifndef CORRADE_BUILD_STATIC - #ifdef CorradeTestSuite_EXPORTS + #if defined(CorradeTestSuite_EXPORTS) || defined(CorradeTestSuiteObjects_EXPORTS) || defined(CorradeTestSuiteTestLib_EXPORTS) #define CORRADE_TESTSUITE_EXPORT CORRADE_VISIBILITY_EXPORT #else #define CORRADE_TESTSUITE_EXPORT CORRADE_VISIBILITY_IMPORT diff --git a/src/Corrade/Utility/CMakeLists.txt b/src/Corrade/Utility/CMakeLists.txt index 55d74e0c0..c8bc14aa4 100644 --- a/src/Corrade/Utility/CMakeLists.txt +++ b/src/Corrade/Utility/CMakeLists.txt @@ -197,16 +197,24 @@ if(CORRADE_WITH_UTILITY) install(FILES ${CorradeUtility_HEADERS} DESTINATION ${CORRADE_INCLUDE_INSTALL_DIR}/Utility) if(CORRADE_BUILD_TESTS) - # Library with graceful assert for testing - add_library(CorradeUtilityTestLib ${SHARED_OR_STATIC} ${CorradeUtility_GracefulAssert_SRCS}) + # Library with graceful assert for testing. If tests need such + # behavior, they should link to CorradeTestSuiteTestLib instead. + # Otherwise the implicitly linked CorradeTestSuite would drag in + # CorradeUtility in addition to CorradeUtilityTestLib, leading to ODR + # violations and making ASan builds fail. + add_library(CorradeUtilityTestLib ${SHARED_OR_STATIC} + $ + ${CorradeUtility_GracefulAssert_SRCS}) + target_include_directories(CorradeUtilityTestLib PUBLIC + $) + target_link_libraries(CorradeUtilityTestLib PUBLIC + $) target_compile_definitions(CorradeUtilityTestLib PRIVATE "CORRADE_GRACEFUL_ASSERT") set_target_properties(CorradeUtilityTestLib PROPERTIES DEBUG_POSTFIX "-d") if(CORRADE_BUILD_STATIC_PIC) set_target_properties(CorradeUtilityTestLib PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() - target_link_libraries(CorradeUtilityTestLib CorradeUtility) - add_subdirectory(Test) endif() diff --git a/src/Corrade/Utility/Test/CMakeLists.txt b/src/Corrade/Utility/Test/CMakeLists.txt index 0ba8c3739..39d9177b4 100644 --- a/src/Corrade/Utility/Test/CMakeLists.txt +++ b/src/Corrade/Utility/Test/CMakeLists.txt @@ -39,10 +39,16 @@ endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/configure.h) -corrade_add_test(UtilityAlgorithmsTest AlgorithmsTest.cpp LIBRARIES CorradeUtilityTestLib) +# In all following corrade_add_test() macros, if a test wants to use +# CorradeUtilityTestLib, it has to link to CorradeTestSuiteTestLib instead. +# Otherwise the implicitly linked CorradeTestSuite would drag in CorradeUtility +# in addition to CorradeUtilityTestLib, leading to ODR violations and making +# ASan builds fail. + +corrade_add_test(UtilityAlgorithmsTest AlgorithmsTest.cpp LIBRARIES CorradeTestSuiteTestLib) target_compile_definitions(UtilityAlgorithmsTest PRIVATE "CORRADE_GRACEFUL_ASSERT") -corrade_add_test(UtilityArgumentsTest ArgumentsTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityArgumentsTest ArgumentsTest.cpp LIBRARIES CorradeTestSuiteTestLib) set_tests_properties(UtilityArgumentsTest PROPERTIES ENVIRONMENT "ARGUMENTSTEST_SIZE=1337;ARGUMENTSTEST_VERBOSE=ON;ARGUMENTSTEST_COLOR=OFF;ARGUMENTSTEST_UNICODE=hýždě") @@ -149,7 +155,7 @@ corrade_add_test(UtilityEndiannessTest EndiannessTest.cpp) corrade_add_test(UtilityErrorStringTest ErrorStringTest.cpp) corrade_add_test(UtilityMurmurHash2Test MurmurHash2Test.cpp) corrade_add_test(UtilityConfigurationTest ConfigurationTest.cpp - LIBRARIES CorradeUtilityTestLib + LIBRARIES CorradeTestSuiteTestLib FILES ConfigurationTestFiles/bom.conf ConfigurationTestFiles/comments.conf @@ -224,14 +230,14 @@ corrade_add_test(UtilityFatalTest FatalTest.cpp) set_tests_properties(UtilityFatalTest PROPERTIES WILL_FAIL ON) corrade_add_test(UtilityJsonTest JsonTest.cpp - LIBRARIES CorradeUtilityTestLib + LIBRARIES CorradeTestSuiteTestLib FILES JsonTestFiles/error.json JsonTestFiles/parse-error.json) target_compile_definitions(UtilityJsonTest PRIVATE "CORRADE_GRACEFUL_ASSERT") target_include_directories(UtilityJsonTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -corrade_add_test(UtilityJsonWriterTest JsonWriterTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityJsonWriterTest JsonWriterTest.cpp LIBRARIES CorradeTestSuiteTestLib) target_include_directories(UtilityJsonWriterTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) corrade_add_test(UtilityMemoryTest MemoryTest.cpp) @@ -269,7 +275,7 @@ if(CORRADE_TARGET_UNIX) endif() target_include_directories(UtilityPathTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -corrade_add_test(UtilityFormatTest FormatTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityFormatTest FormatTest.cpp LIBRARIES CorradeTestSuiteTestLib) target_include_directories(UtilityFormatTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) corrade_add_test(UtilityHashDigestTest HashDigestTest.cpp) @@ -290,12 +296,12 @@ if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_CXX_COMPILER_VERSION VERS CORRADE_CXX_STANDARD 17) endif() -corrade_add_test(UtilityStringTest StringTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityStringTest StringTest.cpp LIBRARIES CorradeTestSuiteTestLib) corrade_add_test(UtilityStringBenchmark StringBenchmark.cpp) corrade_add_test(UtilitySystemTest SystemTest.cpp) corrade_add_test(UtilityTweakableParserTest TweakableParserTest.cpp) corrade_add_test(UtilityTypeTraitsTest TypeTraitsTest.cpp) -corrade_add_test(UtilityUnicodeTest UnicodeTest.cpp LIBRARIES CorradeUtilityTestLib) +corrade_add_test(UtilityUnicodeTest UnicodeTest.cpp LIBRARIES CorradeTestSuiteTestLib) # Compiled-in resource test corrade_add_resource(ResourceTestData ResourceTestFiles/resources.conf) @@ -306,7 +312,7 @@ corrade_add_test(UtilityResourceTest ${ResourceTestData} ${ResourceTestEmptyFileData} ${ResourceTestNothingData} - LIBRARIES CorradeUtilityTestLib + LIBRARIES CorradeTestSuiteTestLib FILES ResourceTestFiles/consequence.bin # Referenced from resources-overriden.conf From 75c786d7b53f3162ad23fecc26371d738b7f7841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 21:18:31 +0200 Subject: [PATCH 55/79] CMake: use dummy.cpp for TestSuite[TestLib] and document what is it for. Goddamit, the iOS build just failing with "libCorradeTestSuiteTestLib.a not found" and nothing else whatsoever is NOT HELPFUL. --- src/Corrade/TestSuite/CMakeLists.txt | 6 ++++-- src/Corrade/Utility/Test/CMakeLists.txt | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Corrade/TestSuite/CMakeLists.txt b/src/Corrade/TestSuite/CMakeLists.txt index d300ac6ed..0b31ca109 100644 --- a/src/Corrade/TestSuite/CMakeLists.txt +++ b/src/Corrade/TestSuite/CMakeLists.txt @@ -73,7 +73,8 @@ endif() # Main TestSuite library add_library(CorradeTestSuite ${SHARED_OR_STATIC} - $) + $ + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details set_target_properties(CorradeTestSuite PROPERTIES DEBUG_POSTFIX "-d") if(NOT CORRADE_BUILD_STATIC) @@ -107,7 +108,8 @@ if(CORRADE_BUILD_TESTS) # CorradeUtilityTestLib, leading to ODR violations and subsequently ASan # complaints. add_library(CorradeTestSuiteTestLib ${SHARED_OR_STATIC} - $) + $ + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details target_include_directories(CorradeTestSuiteTestLib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) set_target_properties(CorradeTestSuiteTestLib PROPERTIES DEBUG_POSTFIX "-d") if(CORRADE_BUILD_STATIC_PIC) diff --git a/src/Corrade/Utility/Test/CMakeLists.txt b/src/Corrade/Utility/Test/CMakeLists.txt index 39d9177b4..6b3742b5e 100644 --- a/src/Corrade/Utility/Test/CMakeLists.txt +++ b/src/Corrade/Utility/Test/CMakeLists.txt @@ -59,10 +59,10 @@ target_include_directories(DebugAssertTestObjects PRIVATE $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp) + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details corrade_add_test(UtilityDebugAssertTest $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp) + ${PROJECT_SOURCE_DIR}/src/dummy.cpp) # XCode workaround, see file comment for details # WILL_FAIL doesn't work for abort() on desktop, test this only on embedded # then. Oh well. Also the tests could be just one executable added multiple # times with different arguments, but corrade_add_test() doesn't support that, @@ -71,55 +71,55 @@ if(CORRADE_TARGET_EMSCRIPTEN OR CORRADE_TARGET_ANDROID) foreach(debug "" "Debug") corrade_add_test(Utility${debug}AssertTestFailAssert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-assert true) set_tests_properties(Utility${debug}AssertTestFailAssert PROPERTIES PASS_REGULAR_EXPRESSION "A should be zero") corrade_add_test(Utility${debug}AssertTestFailConstexprAssert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-constexpr-assert true) set_tests_properties(Utility${debug}AssertTestFailConstexprAssert PROPERTIES PASS_REGULAR_EXPRESSION "b can't be zero") corrade_add_test(Utility${debug}AssertTestFailInternalAssert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert true) set_tests_properties(Utility${debug}AssertTestFailInternalAssert PROPERTIES PASS_REGULAR_EXPRESSION "Assertion b && !_failInternalAssert failed at ") corrade_add_test(Utility${debug}AssertTestFailInternalC___Assert $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-constexpr-assert true) set_tests_properties(Utility${debug}AssertTestFailInternalC___Assert PROPERTIES PASS_REGULAR_EXPRESSION "Assertion b failed at ") corrade_add_test(Utility${debug}AssertTestFailAssertOutput $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-assert-output true) set_tests_properties(Utility${debug}AssertTestFailAssertOutput PROPERTIES PASS_REGULAR_EXPRESSION "foo\\(\\) should succeed") corrade_add_test(Utility${debug}AssertTestFailInternalA___Output $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert-output true) set_tests_properties(Utility${debug}AssertTestFailInternalA___Output PROPERTIES PASS_REGULAR_EXPRESSION "Assertion foo\\(\\) && !_failInternalAssertOutput failed at ") corrade_add_test(Utility${debug}AssertTestFailInter___Expression $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert-expression true) set_tests_properties(Utility${debug}AssertTestFailInter___Expression PROPERTIES PASS_REGULAR_EXPRESSION "Assertion c \\+ \\(_failInternalAssertExpression \\? -3 : 3\\) failed at ") corrade_add_test(Utility${debug}AssertTestFailAssertUnreachable $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-assert-unreachable true) set_tests_properties(Utility${debug}AssertTestFailAssertUnreachable PROPERTIES PASS_REGULAR_EXPRESSION "C should be 3") corrade_add_test(Utility${debug}AssertTestFailInte___Unreachable $ - ${PROJECT_SOURCE_DIR}/src/dummy.cpp + ${PROJECT_SOURCE_DIR}/src/dummy.cpp # XCode workaround, see file comment for details ARGUMENTS --fail-on-internal-assert-unreachable true) set_tests_properties(Utility${debug}AssertTestFailInte___Unreachable PROPERTIES PASS_REGULAR_EXPRESSION "Reached unreachable code at ") From d9cc4a20ddf7b0ae9eed4f47d9cd6eaf9fb76416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 21:35:56 +0200 Subject: [PATCH 56/79] CMake: consistently use PUBLIC in target_link_libraries(). --- doc/snippets/CMakeLists.txt | 16 ++++++++-------- src/Corrade/PluginManager/CMakeLists.txt | 6 +++--- src/Corrade/Utility/Test/CMakeLists.txt | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/snippets/CMakeLists.txt b/doc/snippets/CMakeLists.txt index 77daa5f49..a15e763c9 100644 --- a/doc/snippets/CMakeLists.txt +++ b/doc/snippets/CMakeLists.txt @@ -119,14 +119,14 @@ if(CORRADE_WITH_TESTSUITE AND NOT CORRADE_TARGET_IOS) add_executable(testsuite-benchmark testsuite-benchmark.cpp) add_executable(testsuite-benchmark-custom testsuite-benchmark-custom.cpp) - target_link_libraries(testsuite-basic CorradeTestSuite) - target_link_libraries(testsuite-templated CorradeTestSuite) - target_link_libraries(testsuite-repeated CorradeTestSuite ${CMAKE_THREAD_LIBS_INIT}) - target_link_libraries(testsuite-instanced CorradeTestSuite) - target_link_libraries(testsuite-save-diagnostic CorradeTestSuite) - target_link_libraries(testsuite-iteration CorradeTestSuite) - target_link_libraries(testsuite-benchmark CorradeTestSuite) - target_link_libraries(testsuite-benchmark-custom CorradeTestSuite) + target_link_libraries(testsuite-basic PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-templated PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-repeated PUBLIC CorradeTestSuite ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(testsuite-instanced PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-save-diagnostic PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-iteration PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-benchmark PUBLIC CorradeTestSuite) + target_link_libraries(testsuite-benchmark-custom PUBLIC CorradeTestSuite) endif() if(CORRADE_TARGET_ANDROID) diff --git a/src/Corrade/PluginManager/CMakeLists.txt b/src/Corrade/PluginManager/CMakeLists.txt index 60b510ea8..6804fead8 100644 --- a/src/Corrade/PluginManager/CMakeLists.txt +++ b/src/Corrade/PluginManager/CMakeLists.txt @@ -74,7 +74,7 @@ elseif(CORRADE_BUILD_STATIC_PIC) set_target_properties(CorradePluginManager PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() # Utility also does -ldl now -target_link_libraries(CorradePluginManager CorradeUtility) +target_link_libraries(CorradePluginManager PUBLIC CorradeUtility) install(TARGETS CorradePluginManager RUNTIME DESTINATION ${CORRADE_BINARY_INSTALL_DIR} @@ -93,9 +93,9 @@ if(CORRADE_BUILD_TESTS) if(CORRADE_BUILD_STATIC_PIC) set_target_properties(CorradePluginManagerTestLib PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() - target_link_libraries(CorradePluginManagerTestLib CorradeUtility) + target_link_libraries(CorradePluginManagerTestLib PUBLIC CorradeUtility) if(CORRADE_TARGET_UNIX) - target_link_libraries(CorradePluginManagerTestLib ${CMAKE_DL_LIBS}) + target_link_libraries(CorradePluginManagerTestLib PUBLIC ${CMAKE_DL_LIBS}) endif() add_subdirectory(Test) diff --git a/src/Corrade/Utility/Test/CMakeLists.txt b/src/Corrade/Utility/Test/CMakeLists.txt index 6b3742b5e..a0669b539 100644 --- a/src/Corrade/Utility/Test/CMakeLists.txt +++ b/src/Corrade/Utility/Test/CMakeLists.txt @@ -335,7 +335,7 @@ add_library(ResourceTestDataLib STATIC ${ResourceTestData} target_compile_definitions(ResourceTestDataLib PRIVATE "CORRADE_AUTOMATIC_INITIALIZER=CORRADE_NOOP" "CORRADE_AUTOMATIC_FINALIZER=CORRADE_NOOP") -target_link_libraries(ResourceTestDataLib CorradeUtility) +target_link_libraries(ResourceTestDataLib PUBLIC CorradeUtility) corrade_add_test(UtilityResourceStaticTest ResourceStaticTest.cpp LIBRARIES ResourceTestDataLib FILES From c3acf1cd690a7e97b8249f185cc5142c25219415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 26 Jul 2022 21:36:21 +0200 Subject: [PATCH 57/79] CMake: use the new-style Threads::Threads target. --- doc/snippets/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/snippets/CMakeLists.txt b/doc/snippets/CMakeLists.txt index a15e763c9..e3a0ac4c4 100644 --- a/doc/snippets/CMakeLists.txt +++ b/doc/snippets/CMakeLists.txt @@ -96,6 +96,7 @@ endif() # TODO: causes spurious linker errors on Travis iOS build, so I'm disabling it if(CORRADE_WITH_TESTSUITE AND NOT CORRADE_TARGET_IOS) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) # All define the same symbols so they need to be in separate libs @@ -121,7 +122,7 @@ if(CORRADE_WITH_TESTSUITE AND NOT CORRADE_TARGET_IOS) target_link_libraries(testsuite-basic PUBLIC CorradeTestSuite) target_link_libraries(testsuite-templated PUBLIC CorradeTestSuite) - target_link_libraries(testsuite-repeated PUBLIC CorradeTestSuite ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(testsuite-repeated PUBLIC CorradeTestSuite Threads::Threads) target_link_libraries(testsuite-instanced PUBLIC CorradeTestSuite) target_link_libraries(testsuite-save-diagnostic PUBLIC CorradeTestSuite) target_link_libraries(testsuite-iteration PUBLIC CorradeTestSuite) From 66d7a38e37ba77378f9dde47bbaadc014528c18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 22 Jul 2022 19:09:43 +0200 Subject: [PATCH 58/79] Utility: add helpers for CPU tests. Given that these rely on library-specific macros, I assume each library will eventually have its own copy. I don't know any better solution at the moment. --- src/Corrade/Utility/Test/cpuVariantHelpers.h | 73 ++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/Corrade/Utility/Test/cpuVariantHelpers.h diff --git a/src/Corrade/Utility/Test/cpuVariantHelpers.h b/src/Corrade/Utility/Test/cpuVariantHelpers.h new file mode 100644 index 000000000..271dce35d --- /dev/null +++ b/src/Corrade/Utility/Test/cpuVariantHelpers.h @@ -0,0 +1,73 @@ +#ifndef Corrade_Utility_Test_cpuVariantHelpers_h +#define Corrade_Utility_Test_cpuVariantHelpers_h +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include "Corrade/Cpu.h" +#include "Corrade/Containers/String.h" +#include "Corrade/Containers/StringStl.h" +#include "Corrade/Utility/Assert.h" + +namespace Corrade { namespace Utility { namespace Test { + +template constexpr std::size_t cpuVariantCount(T(&)[size]) { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + return size; + #else + return 1; + #endif +} + +template inline Containers::String cpuVariantName(T& data) { + std::ostringstream out; + Utility::Debug{&out, Utility::Debug::Flag::NoNewlineAtTheEnd} << Utility::Debug::packed << data.features; + return out.str(); +} + +template inline const T& cpuVariantCompiled(const T(&data)[size]) { + const Cpu::Features features = + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + Cpu::runtimeFeatures() + #else + Cpu::compiledFeatures() + #endif + ; + for(std::size_t i = size; i != 0; --i) + if(features >= data[i - 1].features) + return data[i - 1]; + + CORRADE_INTERNAL_ASSERT_UNREACHABLE(); +} + +template inline bool isCpuVariantSupported(T& data) { + return Cpu::runtimeFeatures() >= data.features; +} + +}}} + +#endif From 8dd4a33a82b7fd6dfe1d7116ed0007cc0caf1346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 28 Jul 2022 20:42:51 +0200 Subject: [PATCH 59/79] Containers: SSE2+BMI1 implementation of StringView::find(). --- src/Corrade/Containers/StringView.cpp | 160 +++++++++- src/Corrade/Containers/StringView.h | 4 +- .../Containers/Test/StringViewBenchmark.cpp | 94 +++++- .../Containers/Test/StringViewTest.cpp | 275 +++++++++++++++++- src/Corrade/Utility/CMakeLists.txt | 36 ++- src/Corrade/Utility/visibility.h | 33 +++ 6 files changed, 579 insertions(+), 23 deletions(-) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 09b068865..ab3f185c7 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -30,6 +30,7 @@ #include #include +#include "Corrade/Cpu.h" #include "Corrade/Containers/Array.h" #include "Corrade/Containers/ArrayView.h" #include "Corrade/Containers/GrowableArray.h" @@ -39,6 +40,10 @@ #include "Corrade/Utility/DebugStl.h" #include "Corrade/Utility/Math.h" +#if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) +#include "Corrade/Utility/IntrinsicsAvx.h" /* TZCNT is in AVX headers :( */ +#endif + namespace Corrade { namespace Containers { template BasicStringView::BasicStringView(T* const data, const StringViewFlags flags, std::nullptr_t) noexcept: BasicStringView{data, @@ -143,13 +148,160 @@ const char* stringFindLastString(const char* const data, const std::size_t size, return {}; } -const char* stringFindCharacter(const char* data, const std::size_t size, const char character) { - /* Making a utility function because yet again I'm not sure if null - pointers are allowed and cppreference says nothing about that, so in - case this needs to be patched it's better to have it in a single place */ +namespace { + +/* SIMD implementation of character lookup. Loosely based off + https://docs.rs/memchr/2.3.4/src/memchr/x86/sse2.rs.html, which in turn is + based off https://gms.tf/stdfind-and-memchr-optimizations.html, which at the + time of writing (Jul 2022) uses m.css, so the circle is complete :)) + + The code below is commented, but the core points are the following: + + 1. do as much as possible via aligned loads, + 2. otherwise, do as much as possible via unaligned vector loads even at + the cost of ovelapping with an aligned load, + 3. otherwise, fall back to a smaller vector width (AVX -> SSE) or to a + scalar code + + The 128-bit variant first checks if there's less than 16 bytes. If it is, it + just checks each of them sequentially. Otherwise, with 16 and more bytes, + the following is done: + + +---+ +---+ + | A | | D | + +---+ +---+ + +---+---+---+---+ +---+-- + | B : : : | ... | C | ... + +---+---+---+---+ +---+-- + + A. First it does an unconditional unaligned load of a single vector + (assuming an extra conditional branch would likely be slower than the + unaligned load ovehead), compares all bytes inside to the (broadcasted) + search value and for all bytes that are equal calculates a bitmask (if + 4th and 7th byte is present, the bitmask has bit 4 and 7 set). Then, if + any bit is set, returns the position of the first bit which is the + found index. + B. Next it finds an aligned position. If the vector A was already aligned, + it will start right after, otherwise there may be up to 15 bytes + overlap that'll be checked twice. From the aligned position, to avoid + branching too often, it goes in a batch of four vectors at a time, + checking the result together for all four. Which also helps offset the + extra work from the initial overlap. + C. Once there is less than four vectors left, it goes vector-by-vector, + still doing aligned loads, but branching for every. + D. Once there's less than 16 bytes left, it performs an unaligned load + that may overlap with the previous aligned vector, similarly to the + initial unaligned load A. */ + +#if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) +CORRADE_ENABLE(SSE2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVectorUnaligned(Cpu::Sse2T, const char* at, const __m128i vn1) { + /* _mm_lddqu_si128 is just an alias to _mm_loadu_si128 on all CPUs with + SSSE3+, no reason to use it: https://stackoverflow.com/a/38383624 */ + const __m128i chunk = _mm_loadu_si128(reinterpret_cast(at)); + if(const int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} +CORRADE_ENABLE(SSE2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVector(Cpu::Sse2T, const char* at, const __m128i vn1) { + CORRADE_INTERNAL_DEBUG_ASSERT(reinterpret_cast(at) % 16 == 0); + + const __m128i chunk = _mm_load_si128(reinterpret_cast(at)); + if(const int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} + +CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2|Cpu::Bmi1)) { + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(SSE2,BMI1) -> const char* { + const char* const end = data + size; + + /* If we have less than 16 bytes, do it the stupid way */ + /** @todo SWAR?? */ + if(size < 16) { + for(const char* i = data; i != end; ++i) + if(*i == character) return i; + return {}; + } + + const __m128i vn1 = _mm_set1_epi8(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Sse2, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 16) & ~0xf); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 16 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*16 < end; i += 4*16) { + const __m128i a = _mm_load_si128(reinterpret_cast(i) + 0); + const __m128i b = _mm_load_si128(reinterpret_cast(i) + 1); + const __m128i c = _mm_load_si128(reinterpret_cast(i) + 2); + const __m128i d = _mm_load_si128(reinterpret_cast(i) + 3); + + const __m128i eqa = _mm_cmpeq_epi8(vn1, a); + const __m128i eqb = _mm_cmpeq_epi8(vn1, b); + const __m128i eqc = _mm_cmpeq_epi8(vn1, c); + const __m128i eqd = _mm_cmpeq_epi8(vn1, d); + + const __m128i or1 = _mm_or_si128(eqa, eqb); + const __m128i or2 = _mm_or_si128(eqc, eqd); + const __m128i or3 = _mm_or_si128(or1, or2); + if(_mm_movemask_epi8(or3)) { + if(const int mask = _mm_movemask_epi8(eqa)) + return i + 0*16 + _tzcnt_u32(mask); + if(const int mask = _mm_movemask_epi8(eqb)) + return i + 1*16 + _tzcnt_u32(mask); + if(const int mask = _mm_movemask_epi8(eqc)) + return i + 2*16 + _tzcnt_u32(mask); + if(const int mask = _mm_movemask_epi8(eqd)) + return i + 3*16 + _tzcnt_u32(mask); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 16 <= end; i += 16) + if(const char* const found = findCharacterSingleVector(Cpu::Sse2, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 16 > end); + i = end - 16; + return findCharacterSingleVectorUnaligned(Cpu::Sse2, i, vn1); + } + + return {}; + }; +} +#endif + +CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { + return [](const char* const data, const std::size_t size, const char character) -> const char* { + /* Yet again I'm not sure if null pointers are allowed and cppreference + says nothing about that, so this might need to get patched */ return static_cast(std::memchr(data, character, size)); + }; +} + } +#ifdef CORRADE_TARGET_X86 +CORRADE_UTILITY_CPU_DISPATCHER(stringFindCharacterImplementation, Cpu::Bmi1) +#else +CORRADE_UTILITY_CPU_DISPATCHER(stringFindCharacterImplementation) +#endif +CORRADE_UTILITY_CPU_DISPATCHED(stringFindCharacterImplementation, const char* CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(stringFindCharacter)(const char* data, std::size_t size, char character))({ + return stringFindCharacterImplementation(CORRADE_CPU_SELECT(Cpu::Default))(data, size, character); +}) + const char* stringFindLastCharacter(const char* const data, const std::size_t size, const char character) { /* Linux has a memrchr() function but other OSes not. So let's just do it myself, that way I also don't need to worry about null pointers being diff --git a/src/Corrade/Containers/StringView.h b/src/Corrade/Containers/StringView.h index a3de4e44f..10d145345 100644 --- a/src/Corrade/Containers/StringView.h +++ b/src/Corrade/Containers/StringView.h @@ -35,6 +35,7 @@ #include #include +#include "Corrade/Corrade.h" #include "Corrade/Containers/Containers.h" #include "Corrade/Containers/EnumSet.h" #include "Corrade/Utility/DebugAssert.h" @@ -1270,7 +1271,8 @@ namespace Implementation { (that can't be overloaded) */ CORRADE_UTILITY_EXPORT const char* stringFindString(const char* data, std::size_t size, const char* substring, std::size_t substringSize); CORRADE_UTILITY_EXPORT const char* stringFindLastString(const char* data, std::size_t size, const char* substring, std::size_t substringSize); -CORRADE_UTILITY_EXPORT const char* stringFindCharacter(const char* data, std::size_t size, char character); +CORRADE_UTILITY_EXPORT extern const char* CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(stringFindCharacter)(const char* data, std::size_t size, char character); +CORRADE_UTILITY_CPU_DISPATCHER_DECLARATION(stringFindCharacter) CORRADE_UTILITY_EXPORT const char* stringFindLastCharacter(const char* data, std::size_t size, char character); CORRADE_UTILITY_EXPORT const char* stringFindAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); CORRADE_UTILITY_EXPORT const char* stringFindLastAny(const char* data, std::size_t size, const char* characters, std::size_t characterCount); diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp index 838654be5..95a03ab49 100644 --- a/src/Corrade/Containers/Test/StringViewBenchmark.cpp +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -27,12 +27,15 @@ #include #include +#include "Corrade/Cpu.h" #include "Corrade/Containers/Optional.h" +#include "Corrade/Containers/ArrayView.h" #include "Corrade/Containers/StringView.h" #include "Corrade/Containers/StringStl.h" #include "Corrade/TestSuite/Tester.h" #include "Corrade/Utility/Math.h" #include "Corrade/Utility/Path.h" +#include "Corrade/Utility/Test/cpuVariantHelpers.h" #include "configure.h" @@ -41,6 +44,9 @@ namespace Corrade { namespace Containers { namespace Test { namespace { struct StringViewBenchmark: TestSuite::Tester { explicit StringViewBenchmark(); + void captureImplementations(); + void restoreImplementations(); + /* The "Common" variants test rather the call / preamble / postamble overhead, while the "Rare" variants test the actual vectorized implementation perf */ @@ -74,21 +80,48 @@ struct StringViewBenchmark: TestSuite::Tester { void findLastCharacterRareNaive(); void findLastCharacterRareMemrchr(); void findLastCharacterRareStlString(); + + private: + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + decltype(Implementation::stringFindCharacter) findCharacterImplementation; + #endif }; using namespace Containers::Literals; +const struct { + Cpu::Features features; + std::size_t vectorSize; +} FindCharacterData[]{ + {Cpu::Scalar, 16}, + #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Sse2|Cpu::Bmi1, 16}, + #endif +}; + StringViewBenchmark::StringViewBenchmark() { - addBenchmarks({&StringViewBenchmark::findCharacterCommon, - &StringViewBenchmark::findCharacterCommonNaive, + addInstancedBenchmarks({&StringViewBenchmark::findCharacterCommon}, 100, + Utility::Test::cpuVariantCount(FindCharacterData), + &StringViewBenchmark::captureImplementations, + &StringViewBenchmark::restoreImplementations); + + addBenchmarks({&StringViewBenchmark::findCharacterCommonNaive, &StringViewBenchmark::findCharacterCommonMemchr, - &StringViewBenchmark::findCharacterCommonStlString, + &StringViewBenchmark::findCharacterCommonStlString}, 100); + + addInstancedBenchmarks({&StringViewBenchmark::findCharacterCommonSmall}, 100, + Utility::Test::cpuVariantCount(FindCharacterData), + &StringViewBenchmark::captureImplementations, + &StringViewBenchmark::restoreImplementations); - &StringViewBenchmark::findCharacterCommonSmall, - &StringViewBenchmark::findCharacterCommonSmallMemchr, + addBenchmarks({&StringViewBenchmark::findCharacterCommonSmallMemchr}, 100); - &StringViewBenchmark::findCharacterRare, - &StringViewBenchmark::findCharacterRareNaive, + addInstancedBenchmarks({&StringViewBenchmark::findCharacterRare}, 100, + Utility::Test::cpuVariantCount(FindCharacterData), + &StringViewBenchmark::captureImplementations, + &StringViewBenchmark::restoreImplementations); + + addBenchmarks({&StringViewBenchmark::findCharacterRareNaive, &StringViewBenchmark::findCharacterRareMemchr, &StringViewBenchmark::findCharacterRareStlString, @@ -106,11 +139,34 @@ StringViewBenchmark::StringViewBenchmark() { &StringViewBenchmark::findLastCharacterRareStlString}, 100); } +void StringViewBenchmark::captureImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + findCharacterImplementation = Implementation::stringFindCharacter; + #endif +} + +void StringViewBenchmark::restoreImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + Implementation::stringFindCharacter = findCharacterImplementation; + #endif +} + constexpr std::size_t CommonCharacterCount = 500; constexpr std::size_t RareCharacterCount = 90; constexpr std::size_t CharacterRepeats = 100; void StringViewBenchmark::findCharacterCommon() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); CORRADE_VERIFY(text); @@ -186,13 +242,24 @@ void StringViewBenchmark::findCharacterCommonStlString() { } void StringViewBenchmark::findCharacterCommonSmall() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); CORRADE_VERIFY(text); std::size_t count = 0; CORRADE_BENCHMARK(CharacterRepeats) { StringView a = *text; - while(StringView found = a.prefix(Utility::min(std::size_t{15}, a.size())).find(' ')) { + while(StringView found = a.prefix(Utility::min(data.vectorSize - 1, a.size())).find(' ')) { ++count; a = a.suffix(found.end()); } @@ -219,6 +286,17 @@ void StringViewBenchmark::findCharacterCommonSmallMemchr() { } void StringViewBenchmark::findCharacterRare() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + Containers::Optional text = Utility::Path::readString(Utility::Path::join(CONTAINERS_TEST_DIR, "StringTestFiles/lorem-ipsum.txt")); CORRADE_VERIFY(text); *text = *text*10; diff --git a/src/Corrade/Containers/Test/StringViewTest.cpp b/src/Corrade/Containers/Test/StringViewTest.cpp index d6eb5cd3e..e3ec79642 100644 --- a/src/Corrade/Containers/Test/StringViewTest.cpp +++ b/src/Corrade/Containers/Test/StringViewTest.cpp @@ -26,12 +26,16 @@ #include +#include "Corrade/Cpu.h" #include "Corrade/Containers/Array.h" #include "Corrade/Containers/StaticArray.h" #include "Corrade/Containers/StringView.h" #include "Corrade/TestSuite/Tester.h" #include "Corrade/TestSuite/Compare/Container.h" +#include "Corrade/TestSuite/Compare/Numeric.h" #include "Corrade/Utility/DebugStl.h" /** @todo remove once Debug is stream-free */ +#include "Corrade/Utility/Memory.h" +#include "Corrade/Utility/Test/cpuVariantHelpers.h" namespace { @@ -95,6 +99,9 @@ namespace Test { namespace { struct StringViewTest: TestSuite::Tester { explicit StringViewTest(); + void captureImplementations(); + void restoreImplementations(); + template void constructDefault(); void constructDefaultConstexpr(); template void construct(); @@ -173,6 +180,10 @@ struct StringViewTest: TestSuite::Tester { void findStringMultipleOccurences(); void findStringWhole(); void findCharacter(); + void findCharacterAligned(); + void findCharacterUnaligned(); + void findCharacterUnalignedLessThanTwoVectors(); + void findCharacterUnalignedLessThanOneVector(); void findEmpty(); void findFlags(); void findOr(); @@ -198,6 +209,19 @@ struct StringViewTest: TestSuite::Tester { void debugFlag(); void debugFlags(); void debug(); + + private: + decltype(Implementation::stringFindCharacter) findCharacterImplementation; +}; + +const struct { + Cpu::Features features; + std::size_t vectorSize; +} FindCharacterData[]{ + {Cpu::Scalar, 16}, + #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Sse2|Cpu::Bmi1, 16}, + #endif }; StringViewTest::StringViewTest() { @@ -279,9 +303,18 @@ StringViewTest::StringViewTest() { &StringViewTest::findString, &StringViewTest::findStringMultipleOccurences, - &StringViewTest::findStringWhole, - &StringViewTest::findCharacter, - &StringViewTest::findEmpty, + &StringViewTest::findStringWhole}); + + addInstancedTests({&StringViewTest::findCharacter, + &StringViewTest::findCharacterAligned, + &StringViewTest::findCharacterUnaligned, + &StringViewTest::findCharacterUnalignedLessThanTwoVectors, + &StringViewTest::findCharacterUnalignedLessThanOneVector}, + Utility::Test::cpuVariantCount(FindCharacterData), + &StringViewTest::captureImplementations, + &StringViewTest::restoreImplementations); + + addTests({&StringViewTest::findEmpty, &StringViewTest::findFlags, &StringViewTest::findOr, @@ -318,6 +351,18 @@ template<> struct NameFor { static const char* name() { return "MutableStringView"; } }; +void StringViewTest::captureImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + findCharacterImplementation = Implementation::stringFindCharacter; + #endif +} + +void StringViewTest::restoreImplementations() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + Implementation::stringFindCharacter = findCharacterImplementation; + #endif +} + template void StringViewTest::constructDefault() { setTestCaseTemplateName(NameFor::name()); @@ -1592,6 +1637,17 @@ void StringViewTest::findStringWhole() { } void StringViewTest::findCharacter() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + StringView a = "hello cursed\0world!"_s; /* Single character at the start */ @@ -1645,6 +1701,219 @@ void StringViewTest::findCharacter() { } } +void StringViewTest::findCharacterAligned() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Also, aligned, with 12 vectors + in total, corresponding to the code paths: + + +----+ +----+----+----+----+ +----+----+----+ + |deef| ! gg : hh :i i: jj | |k | ll | m| + +----+ +----+----+----+----+ +----+----+----+ + */ + Containers::Array a; + if(data.vectorSize == 16) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4*2 + 3)); + else if(data.vectorSize == 32) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4*2 + 3)); + else CORRADE_INTERNAL_ASSERT_UNREACHABLE(); + MutableStringView string = arrayView(a); + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::Aligned); + + /* First vector is treated separately. It should pick the first found of + the two; test also the very first and very last. */ + string[0] = 'd'; + string[7] = 'e'; + string[data.vectorSize - 7] = 'e'; + string[data.vectorSize - 1] = 'f'; + CORRADE_COMPARE(string.find('d').data() - string.data(), 0); + CORRADE_COMPARE(string.find('e').data() - string.data(), 7); + CORRADE_COMPARE(string.find('f').data() - string.data(), data.vectorSize - 1); + + /* Then it's four vectors at a time. First four would be empty, second four + would have the data. Test each of the four separately. For each it + should pick the first found of the two. */ + string[data.vectorSize*5 + 3] = 'g'; + string[data.vectorSize*6 - 3] = 'g'; + string[data.vectorSize*6 + 7] = 'h'; + string[data.vectorSize*7 - 7] = 'h'; + string[data.vectorSize*7 + 0] = 'i'; + string[data.vectorSize*8 - 1] = 'i'; + string[data.vectorSize*8 + 2] = 'j'; + string[data.vectorSize*9 - 2] = 'j'; + CORRADE_COMPARE(string.find('g').data() - string.data(), data.vectorSize*5 + 3); + CORRADE_COMPARE(string.find('h').data() - string.data(), data.vectorSize*6 + 7); + CORRADE_COMPARE(string.find('i').data() - string.data(), data.vectorSize*7 + 0); + CORRADE_COMPARE(string.find('j').data() - string.data(), data.vectorSize*8 + 2); + + /* Last less-than-four vectors are again treated separately. Again, for + each it should pick the last found of the two; test also the very first + and very last of the range. */ + string[data.vectorSize* 9 + 0] = 'k'; + string[data.vectorSize*10 + 4] = 'l'; + string[data.vectorSize*11 - 4] = 'l'; + string[data.vectorSize*12 - 1] = 'm'; + CORRADE_COMPARE(string.find('k').data() - string.data(), data.vectorSize*9 + 0); + CORRADE_COMPARE(string.find('l').data() - string.data(), data.vectorSize*10 + 4); + CORRADE_COMPARE(string.find('m').data() - string.data(), data.vectorSize*12 - 1); + + /* A character that's not found should be handled properly even after all + these complex code paths */ + CORRADE_VERIFY(!string.find('n')); +} + +void StringViewTest::findCharacterUnaligned() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Also, aligned, but then slicing: + - the first unaligned vector having all bytes but one overlapping with + the four-at-a-time block + - there being just one four-at-a-time block (the if() branch that skips + the block was sufficiently tested in firstCharacterAligned()) + - there being just one full vector after, and the last unaligned vector + again overlapping with all but one byte with it + + +----+ +----+ + |f | | j| + +----+ +----+ + +----+----+----+----+----+ + |g : : : h|i | + +----+----+----+----+----+ + */ + Containers::Array a; + if(data.vectorSize == 16) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4 + 2)); + else if(data.vectorSize == 32) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*(1 + 4 + 2)); + else CORRADE_INTERNAL_ASSERT_UNREACHABLE(); + MutableStringView string = a.slice(data.vectorSize - 1, a.size() - (data.vectorSize - 1)); + CORRADE_COMPARE(string.size(), data.vectorSize*5 + 2); + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::NotAligned); + + /* First byte should be handled by the initial unaligned check */ + string[0] = 'f'; + string[data.vectorSize - 1] = 'f'; + CORRADE_COMPARE(string.find('f').data() - string.data(), 0); + + /* The four-vectors-at-a-time should handle the aligned middle portion. + Test just the very first and very last of the aligned range. */ + string[data.vectorSize*0 + 1] = 'g'; + string[data.vectorSize*4 + 0] = 'h'; + CORRADE_COMPARE_AS(string.data() + 1, data.vectorSize, + TestSuite::Compare::Aligned); + CORRADE_COMPARE(string.find('g').data() - string.data(), data.vectorSize*0 + 1); + CORRADE_COMPARE(string.find('h').data() - string.data(), data.vectorSize*4 + 0); + + /* The byte right after the aligned block is handled by the "less than + four vectors" block */ + string[data.vectorSize*4 - 1] = 'i'; + CORRADE_COMPARE_AS(string.data() + data.vectorSize*4 + 1, data.vectorSize, + TestSuite::Compare::Aligned); + CORRADE_COMPARE(string.find('i').data() - string.data(), data.vectorSize*4 - 1); + + /* Last byte should be handled by the final unaligned check */ + string[string.size() - 1] = 'j'; + CORRADE_COMPARE(string.find('j').data() - string.data(), data.vectorSize*5 + 1); + + /* A character that's not found should be handled properly even after all + these complex code paths */ + CORRADE_VERIFY(!string.find('k')); +} + +void StringViewTest::findCharacterUnalignedLessThanTwoVectors() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Also, aligned, but then slicing + so there's just two unaligned blocks overlapping in a single byte: + + +----+ + |f | + +----+ + +----+ + |g ! + +----+ */ + Containers::Array a; + if(data.vectorSize == 16) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*3); + else if(data.vectorSize == 32) + a = Utility::allocateAligned(Corrade::ValueInit, data.vectorSize*3); + else CORRADE_INTERNAL_ASSERT_UNREACHABLE(); + MutableStringView string = a.slice(2, 2 + data.vectorSize*2 - 1); + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::NotAligned); + + /* First byte should be handled by the initial unaligned check */ + string[0] = 'f'; + CORRADE_COMPARE(string.find('f').data() - string.data(), 0); + + /* Last byte should be handled by the final unaligned check */ + string[string.size() - 1] = 'g'; + CORRADE_COMPARE(string.find('g').data() - string.data(), data.vectorSize*2 - 2); + + /* A character that's not found should be handled properly here as well */ + CORRADE_VERIFY(!string.find('h')); +} + +void StringViewTest::findCharacterUnalignedLessThanOneVector() { + #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH + auto&& data = FindCharacterData[testCaseInstanceId()]; + Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); + #else + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + #endif + setTestCaseDescription(Utility::Test::cpuVariantName(data)); + + if(!Utility::Test::isCpuVariantSupported(data)) + CORRADE_SKIP("CPU features not supported"); + + /* Allocating an array to not have it null-terminated or SSO'd in order to + trigger ASan if the algorithm goes OOB. Deliberately pick an unaligned + pointer even though it shouldn't matter here. It should pick the first + found of the two. */ + Containers::Array a{Corrade::ValueInit, data.vectorSize}; + MutableStringView string = a.exceptPrefix(1); + string[7] = 'f'; + string[data.vectorSize/2 + 1] = 'f'; + CORRADE_COMPARE_AS(string.data(), data.vectorSize, + TestSuite::Compare::NotAligned); + CORRADE_COMPARE(string.find('f').data() - string.data(), 7); + + /* A character that's not found should be handled properly here as well */ + CORRADE_VERIFY(!string.find('g')); +} + void StringViewTest::findEmpty() { /* Finding an empty string inside a string should return a zero-sized view to the first byte */ diff --git a/src/Corrade/Utility/CMakeLists.txt b/src/Corrade/Utility/CMakeLists.txt index c8bc14aa4..9a357c441 100644 --- a/src/Corrade/Utility/CMakeLists.txt +++ b/src/Corrade/Utility/CMakeLists.txt @@ -34,7 +34,6 @@ if(CORRADE_WITH_UTILITY) Configuration.cpp ConfigurationValue.cpp MurmurHash2.cpp - Path.cpp Sha1.cpp System.cpp @@ -58,6 +57,13 @@ if(CORRADE_WITH_UTILITY) ../Containers/String.cpp ../Containers/StringView.cpp) + # Files that directly or indirectly use CPU dispatch, such as calling + # Containers::String::contains(). The list gets added to either + # CorradeUtility_GracefulAssert_SRCS or CorradeUtility_SRCS below based on + # whether tests use a different CPU dispatch than the main library. + set(CorradeUtility_CpuDispatch_SRCS + Path.cpp) + set(CorradeUtility_HEADERS Algorithms.h Arguments.h @@ -127,7 +133,8 @@ if(CORRADE_WITH_UTILITY) if(CORRADE_TARGET_UNIX OR (CORRADE_TARGET_WINDOWS AND NOT CORRADE_TARGET_WINDOWS_RT) OR CORRADE_TARGET_EMSCRIPTEN) list(APPEND CorradeUtility_SRCS FileWatcher.cpp - Tweakable.cpp + Tweakable.cpp) + list(APPEND CorradeUtility_CpuDispatch_SRCS TweakableParser.cpp) list(APPEND CorradeUtility_HEADERS FileWatcher.h @@ -149,6 +156,16 @@ if(CORRADE_WITH_UTILITY) list(APPEND CorradeUtility_PRIVATE_HEADERS Implementation/WindowsWeakSymbol.h) endif() + # If the tests are built with pointer dispatch but the main libraries + # aren't, we have to compile select files separately for each. Otherwise + # some code would expect a function symbol to be exist and some expect a + # function pointer symbol. + if(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH AND (NOT CORRADE_BUILD_CPU_RUNTIME_DISPATCH OR CORRADE_CPU_USE_IFUNC)) + list(APPEND CorradeUtility_GracefulAssert_SRCS ${CorradeUtility_CpuDispatch_SRCS}) + else() + list(APPEND CorradeUtility_SRCS ${CorradeUtility_CpuDispatch_SRCS}) + endif() + # Objects shared between main and test library add_library(CorradeUtilityObjects OBJECT ${CorradeUtility_SRCS} @@ -197,11 +214,12 @@ if(CORRADE_WITH_UTILITY) install(FILES ${CorradeUtility_HEADERS} DESTINATION ${CORRADE_INCLUDE_INSTALL_DIR}/Utility) if(CORRADE_BUILD_TESTS) - # Library with graceful assert for testing. If tests need such - # behavior, they should link to CorradeTestSuiteTestLib instead. - # Otherwise the implicitly linked CorradeTestSuite would drag in - # CorradeUtility in addition to CorradeUtilityTestLib, leading to ODR - # violations and making ASan builds fail. + # Library with graceful assert and function-pointer-based CPU dispatch + # for testing. If tests need such behavior, they should link to + # CorradeTestSuiteTestLib instead. Otherwise the implicitly linked + # CorradeTestSuite would drag in CorradeUtility in addition to + # CorradeUtilityTestLib, leading to ODR violations and making ASan + # builds fail. add_library(CorradeUtilityTestLib ${SHARED_OR_STATIC} $ ${CorradeUtility_GracefulAssert_SRCS}) @@ -210,6 +228,9 @@ if(CORRADE_WITH_UTILITY) target_link_libraries(CorradeUtilityTestLib PUBLIC $) target_compile_definitions(CorradeUtilityTestLib PRIVATE "CORRADE_GRACEFUL_ASSERT") + if(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH) + target_compile_definitions(CorradeUtilityTestLib PUBLIC "CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH") + endif() set_target_properties(CorradeUtilityTestLib PROPERTIES DEBUG_POSTFIX "-d") if(CORRADE_BUILD_STATIC_PIC) set_target_properties(CorradeUtilityTestLib PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -243,6 +264,7 @@ if(NOT CMAKE_CROSSCOMPILING) Implementation/ErrorString.cpp + ../Cpu.cpp ../Containers/String.cpp ../Containers/StringView.cpp) if(CORRADE_TARGET_WINDOWS) diff --git a/src/Corrade/Utility/visibility.h b/src/Corrade/Utility/visibility.h index dbdf160da..ab75a1d0e 100644 --- a/src/Corrade/Utility/visibility.h +++ b/src/Corrade/Utility/visibility.h @@ -45,4 +45,37 @@ #define CORRADE_UTILITY_LOCAL #endif +/* Function-pointer-based CPU dispatch, exposing also the runtime dispatcher + implementation */ +#if defined(CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH) || (defined(CORRADE_BUILD_CPU_RUNTIME_DISPATCH) && !defined(CORRADE_CPU_USE_IFUNC)) + #define CORRADE_UTILITY_CPU_DISPATCHER_DECLARATION(name) \ + CORRADE_UTILITY_EXPORT decltype(name) name ## Implementation(Cpu::Features); + #define CORRADE_UTILITY_CPU_DISPATCHER(...) CORRADE_CPU_DISPATCHER(__VA_ARGS__) + #define CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(name) (*name) + #define CORRADE_UTILITY_CPU_DISPATCHED(dispatcher, ...) \ + CORRADE_CPU_DISPATCHED_POINTER(dispatcher, __VA_ARGS__) CORRADE_NOOP + #define CORRADE_UTILITY_CPU_MAYBE_UNUSED + +/* IFUNC or compile-time CPU dispatch, runtime dispatcher is not exposed */ +#else + #define CORRADE_UTILITY_CPU_DISPATCHER_DECLARATION(name) + #define CORRADE_UTILITY_CPU_DISPATCHED_DECLARATION(name) (name) + /* Runtime dispatcher implementation is either hidden or not present at + all */ + #if defined(CORRADE_BUILD_CPU_RUNTIME_DISPATCH) && defined(CORRADE_CPU_USE_IFUNC) + #define CORRADE_UTILITY_CPU_DISPATCHER(...) \ + namespace { CORRADE_CPU_DISPATCHER(__VA_ARGS__) } + #define CORRADE_UTILITY_CPU_DISPATCHED(dispatcher, ...) \ + CORRADE_CPU_DISPATCHED_IFUNC(dispatcher, __VA_ARGS__) CORRADE_NOOP + #define CORRADE_UTILITY_CPU_MAYBE_UNUSED + #elif !defined(CORRADE_BUILD_CPU_RUNTIME_DISPATCH) + #define CORRADE_UTILITY_CPU_DISPATCHER(...) + #define CORRADE_UTILITY_CPU_DISPATCHED(dispatcher, ...) \ + __VA_ARGS__ CORRADE_PASSTHROUGH + #define CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_UNUSED + #else + #error mosra messed up! + #endif +#endif + #endif From 4552812dd3eac80c8deda57ace9f03066887c77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 28 Jul 2022 20:11:45 +0200 Subject: [PATCH 60/79] Containers: AVX2+BMI1 variant of String::find(). See? This is what it takes to add a new variant and have it tested. --- src/Corrade/Containers/StringView.cpp | 98 ++++++++++++++++++- .../Containers/Test/StringViewBenchmark.cpp | 3 + .../Containers/Test/StringViewTest.cpp | 3 + 3 files changed, 102 insertions(+), 2 deletions(-) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index ab3f185c7..9a94df203 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -40,7 +40,7 @@ #include "Corrade/Utility/DebugStl.h" #include "Corrade/Utility/Math.h" -#if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) +#if (defined(CORRADE_ENABLE_SSE2) || defined(CORRADE_ENABLE_AVX)) && defined(CORRADE_ENABLE_BMI1) #include "Corrade/Utility/IntrinsicsAvx.h" /* TZCNT is in AVX headers :( */ #endif @@ -191,7 +191,13 @@ namespace { still doing aligned loads, but branching for every. D. Once there's less than 16 bytes left, it performs an unaligned load that may overlap with the previous aligned vector, similarly to the - initial unaligned load A. */ + initial unaligned load A. + + The 256-bit variant is mostly just about expanding from 16 bytes at a time + to 32 bytes at a time. The only difference is that instead of doing a + scalar fallback for less than 32 bytes, it delegates to the 128-bit + variant --- effectively performing the lookup with either two overlapping + 16-byte vectors (or falling back to scalar for less than 16 bytes). */ #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) CORRADE_ENABLE(SSE2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVectorUnaligned(Cpu::Sse2T, const char* at, const __m128i vn1) { @@ -283,6 +289,94 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay(at)); + if(const int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} +CORRADE_ENABLE(AVX2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVector(Cpu::Avx2T, const char* at, const __m256i vn1) { + CORRADE_INTERNAL_DEBUG_ASSERT(reinterpret_cast(at) % 32 == 0); + + const __m256i chunk = _mm256_load_si256(reinterpret_cast(at)); + if(const int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk, vn1))) + return at + _tzcnt_u32(mask); + return {}; +} + +CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Bmi1)) { + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(AVX2,BMI1) -> const char* { + const char* const end = data + size; + + /* If we have less than 32 bytes, fall back to the SSE variant */ + /** @todo deinline it here? */ + if(size < 32) + return stringFindCharacterImplementation(CORRADE_CPU_SELECT(Cpu::Sse2|Cpu::Bmi1))(data, size, character); + + const __m256i vn1 = _mm256_set1_epi8(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + /** @todo not great, slower than calling SSE directly :( */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Avx2, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 32) & ~0x1f); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 32 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*32 < end; i += 4*32) { + const __m256i a = _mm256_load_si256(reinterpret_cast(i) + 0); + const __m256i b = _mm256_load_si256(reinterpret_cast(i) + 1); + const __m256i c = _mm256_load_si256(reinterpret_cast(i) + 2); + const __m256i d = _mm256_load_si256(reinterpret_cast(i) + 3); + + const __m256i eqa = _mm256_cmpeq_epi8(vn1, a); + const __m256i eqb = _mm256_cmpeq_epi8(vn1, b); + const __m256i eqc = _mm256_cmpeq_epi8(vn1, c); + const __m256i eqd = _mm256_cmpeq_epi8(vn1, d); + + const __m256i or1 = _mm256_or_si256(eqa, eqb); + const __m256i or2 = _mm256_or_si256(eqc, eqd); + const __m256i or3 = _mm256_or_si256(or1, or2); + if(_mm256_movemask_epi8(or3)) { + if(const int mask = _mm256_movemask_epi8(eqa)) + return i + 0*32 + _tzcnt_u32(mask); + if(const int mask = _mm256_movemask_epi8(eqb)) + return i + 1*32 + _tzcnt_u32(mask); + if(const int mask = _mm256_movemask_epi8(eqc)) + return i + 2*32 + _tzcnt_u32(mask); + if(const int mask = _mm256_movemask_epi8(eqd)) + return i + 3*32 + _tzcnt_u32(mask); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 32 <= end; i += 32) + if(const char* const found = findCharacterSingleVector(Cpu::Avx2, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 32 > end); + i = end - 32; + return findCharacterSingleVectorUnaligned(Cpu::Avx2, i, vn1); + } + + return {}; + }; +} +#endif + CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { return [](const char* const data, const std::size_t size, const char character) -> const char* { /* Yet again I'm not sure if null pointers are allowed and cppreference diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp index 95a03ab49..4fb26cb26 100644 --- a/src/Corrade/Containers/Test/StringViewBenchmark.cpp +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -97,6 +97,9 @@ const struct { #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) {Cpu::Sse2|Cpu::Bmi1, 16}, #endif + #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Avx2|Cpu::Bmi1, 32}, + #endif }; StringViewBenchmark::StringViewBenchmark() { diff --git a/src/Corrade/Containers/Test/StringViewTest.cpp b/src/Corrade/Containers/Test/StringViewTest.cpp index e3ec79642..bf6000057 100644 --- a/src/Corrade/Containers/Test/StringViewTest.cpp +++ b/src/Corrade/Containers/Test/StringViewTest.cpp @@ -222,6 +222,9 @@ const struct { #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) {Cpu::Sse2|Cpu::Bmi1, 16}, #endif + #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Avx2|Cpu::Bmi1, 32}, + #endif }; StringViewTest::StringViewTest() { From 07b20279bcd5a59c60a6425e718a6358c538bea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 28 Jul 2022 23:07:18 +0200 Subject: [PATCH 61/79] Containers: benchmark both 15 and 31 bytes in AVX2 StringView::find(). The 31 bytes case is significantly faster due to this falling back to 2 overlapping SSE2 vectors, but we're interested in the scalar fallback too. --- .../Containers/Test/StringViewBenchmark.cpp | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp index 4fb26cb26..6339cc8aa 100644 --- a/src/Corrade/Containers/Test/StringViewBenchmark.cpp +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -33,6 +33,7 @@ #include "Corrade/Containers/StringView.h" #include "Corrade/Containers/StringStl.h" #include "Corrade/TestSuite/Tester.h" +#include "Corrade/Utility/Format.h" #include "Corrade/Utility/Math.h" #include "Corrade/Utility/Path.h" #include "Corrade/Utility/Test/cpuVariantHelpers.h" @@ -91,14 +92,27 @@ using namespace Containers::Literals; const struct { Cpu::Features features; - std::size_t vectorSize; } FindCharacterData[]{ - {Cpu::Scalar, 16}, + {Cpu::Scalar}, #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) - {Cpu::Sse2|Cpu::Bmi1, 16}, + {Cpu::Sse2|Cpu::Bmi1}, #endif #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) - {Cpu::Avx2|Cpu::Bmi1, 32}, + {Cpu::Avx2|Cpu::Bmi1}, + #endif +}; + +const struct { + Cpu::Features features; + std::size_t size; +} FindCharacterSmallData[]{ + {Cpu::Scalar, 15}, + #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Sse2|Cpu::Bmi1, 15}, + #endif + #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) + {Cpu::Avx2|Cpu::Bmi1, 15}, + {Cpu::Avx2|Cpu::Bmi1, 31}, #endif }; @@ -113,7 +127,7 @@ StringViewBenchmark::StringViewBenchmark() { &StringViewBenchmark::findCharacterCommonStlString}, 100); addInstancedBenchmarks({&StringViewBenchmark::findCharacterCommonSmall}, 100, - Utility::Test::cpuVariantCount(FindCharacterData), + Utility::Test::cpuVariantCount(FindCharacterSmallData), &StringViewBenchmark::captureImplementations, &StringViewBenchmark::restoreImplementations); @@ -246,12 +260,12 @@ void StringViewBenchmark::findCharacterCommonStlString() { void StringViewBenchmark::findCharacterCommonSmall() { #ifdef CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH - auto&& data = FindCharacterData[testCaseInstanceId()]; + auto&& data = FindCharacterSmallData[testCaseInstanceId()]; Implementation::stringFindCharacter = Implementation::stringFindCharacterImplementation(data.features); #else - auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterData); + auto&& data = Utility::Test::cpuVariantCompiled(FindCharacterSmallData); #endif - setTestCaseDescription(Utility::Test::cpuVariantName(data)); + setTestCaseDescription(Utility::format("{}, {} bytes", Utility::Test::cpuVariantName(data), data.size)); if(!Utility::Test::isCpuVariantSupported(data)) CORRADE_SKIP("CPU features not supported"); @@ -262,13 +276,12 @@ void StringViewBenchmark::findCharacterCommonSmall() { std::size_t count = 0; CORRADE_BENCHMARK(CharacterRepeats) { StringView a = *text; - while(StringView found = a.prefix(Utility::min(data.vectorSize - 1, a.size())).find(' ')) { + while(StringView found = a.prefix(Utility::min(data.size, a.size())).find(' ')) { ++count; a = a.suffix(found.end()); } } - CORRADE_COMPARE(count, CommonCharacterCount*CharacterRepeats); } From 71fa0f13d9346626e57c8ec7c74ef00047fe2b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 30 Jul 2022 19:17:17 +0200 Subject: [PATCH 62/79] Containers: WASM SIMD128 variant of String::find(). Currently it unconditionally adds -msimd128 when building the test. I expect this to bring immense pain later. --- src/Corrade/Containers/StringView.cpp | 90 ++++++++++++++++++- .../Containers/Test/StringViewBenchmark.cpp | 6 ++ .../Containers/Test/StringViewTest.cpp | 3 + src/Corrade/Utility/CMakeLists.txt | 15 ++-- 4 files changed, 107 insertions(+), 7 deletions(-) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 9a94df203..d86ab7469 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -43,6 +43,9 @@ #if (defined(CORRADE_ENABLE_SSE2) || defined(CORRADE_ENABLE_AVX)) && defined(CORRADE_ENABLE_BMI1) #include "Corrade/Utility/IntrinsicsAvx.h" /* TZCNT is in AVX headers :( */ #endif +#ifdef CORRADE_ENABLE_SIMD128 +#include +#endif namespace Corrade { namespace Containers { @@ -197,7 +200,10 @@ namespace { to 32 bytes at a time. The only difference is that instead of doing a scalar fallback for less than 32 bytes, it delegates to the 128-bit variant --- effectively performing the lookup with either two overlapping - 16-byte vectors (or falling back to scalar for less than 16 bytes). */ + 16-byte vectors (or falling back to scalar for less than 16 bytes). + + The WASM variant is mostly a direct translation of the x86 variant, except + as noted in code comments. */ #if defined(CORRADE_ENABLE_SSE2) && defined(CORRADE_ENABLE_BMI1) CORRADE_ENABLE(SSE2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleVectorUnaligned(Cpu::Sse2T, const char* at, const __m128i vn1) { @@ -377,6 +383,88 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Simd128)) { + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE_SIMD128 -> const char* { + const char* const end = data + size; + + /* If we have less than 16 bytes, do it the stupid way */ + /** @todo SWAR?? */ + if(size < 16) { + for(const char* i = data; i != end; ++i) + if(*i == character) return i; + return {}; + } + + const v128_t vn1 = wasm_i8x16_splat(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Simd128, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 16) & ~0xf); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 16 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*16 < end; i += 4*16) { + const v128_t a = wasm_v128_load(reinterpret_cast(i) + 0); + const v128_t b = wasm_v128_load(reinterpret_cast(i) + 1); + const v128_t c = wasm_v128_load(reinterpret_cast(i) + 2); + const v128_t d = wasm_v128_load(reinterpret_cast(i) + 3); + + const v128_t eqa = wasm_i8x16_eq(vn1, a); + const v128_t eqb = wasm_i8x16_eq(vn1, b); + const v128_t eqc = wasm_i8x16_eq(vn1, c); + const v128_t eqd = wasm_i8x16_eq(vn1, d); + + const v128_t or1 = wasm_v128_or(eqa, eqb); + const v128_t or2 = wasm_v128_or(eqc, eqd); + const v128_t or3 = wasm_v128_or(or1, or2); + if(wasm_i8x16_bitmask(or3)) { + if(const int mask = wasm_i8x16_bitmask(eqa)) + return i + 0*16 + __builtin_ctz(mask); + if(const int mask = wasm_i8x16_bitmask(eqb)) + return i + 1*16 + __builtin_ctz(mask); + if(const int mask = wasm_i8x16_bitmask(eqc)) + return i + 2*16 + __builtin_ctz(mask); + if(const int mask = wasm_i8x16_bitmask(eqd)) + return i + 3*16 + __builtin_ctz(mask); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 16 <= end; i += 16) + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Simd128, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 16 > end); + i = end - 16; + return findCharacterSingleVectorUnaligned(Cpu::Simd128, i, vn1); + } + + return {}; + }; +} +#endif + CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Scalar)) { return [](const char* const data, const std::size_t size, const char character) -> const char* { /* Yet again I'm not sure if null pointers are allowed and cppreference diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp index 6339cc8aa..f55acab51 100644 --- a/src/Corrade/Containers/Test/StringViewBenchmark.cpp +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -100,6 +100,9 @@ const struct { #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) {Cpu::Avx2|Cpu::Bmi1}, #endif + #ifdef CORRADE_ENABLE_SIMD128 + {Cpu::Simd128}, + #endif }; const struct { @@ -114,6 +117,9 @@ const struct { {Cpu::Avx2|Cpu::Bmi1, 15}, {Cpu::Avx2|Cpu::Bmi1, 31}, #endif + #ifdef CORRADE_ENABLE_SIMD128 + {Cpu::Simd128, 15}, + #endif }; StringViewBenchmark::StringViewBenchmark() { diff --git a/src/Corrade/Containers/Test/StringViewTest.cpp b/src/Corrade/Containers/Test/StringViewTest.cpp index bf6000057..c4f38033f 100644 --- a/src/Corrade/Containers/Test/StringViewTest.cpp +++ b/src/Corrade/Containers/Test/StringViewTest.cpp @@ -225,6 +225,9 @@ const struct { #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) {Cpu::Avx2|Cpu::Bmi1, 32}, #endif + #ifdef CORRADE_ENABLE_SIMD128 + {Cpu::Simd128, 16}, + #endif }; StringViewTest::StringViewTest() { diff --git a/src/Corrade/Utility/CMakeLists.txt b/src/Corrade/Utility/CMakeLists.txt index 9a357c441..5036ad2e2 100644 --- a/src/Corrade/Utility/CMakeLists.txt +++ b/src/Corrade/Utility/CMakeLists.txt @@ -214,12 +214,12 @@ if(CORRADE_WITH_UTILITY) install(FILES ${CorradeUtility_HEADERS} DESTINATION ${CORRADE_INCLUDE_INSTALL_DIR}/Utility) if(CORRADE_BUILD_TESTS) - # Library with graceful assert and function-pointer-based CPU dispatch - # for testing. If tests need such behavior, they should link to - # CorradeTestSuiteTestLib instead. Otherwise the implicitly linked - # CorradeTestSuite would drag in CorradeUtility in addition to - # CorradeUtilityTestLib, leading to ODR violations and making ASan - # builds fail. + # Library with graceful assert, function-pointer-based CPU dispatch + # and WASM SIMD128 for testing. If tests need such behavior, they + # should link to CorradeTestSuiteTestLib instead. Otherwise the + # implicitly linked CorradeTestSuite would drag in CorradeUtility in + # addition to CorradeUtilityTestLib, leading to ODR violations and + # making ASan builds fail. add_library(CorradeUtilityTestLib ${SHARED_OR_STATIC} $ ${CorradeUtility_GracefulAssert_SRCS}) @@ -230,6 +230,9 @@ if(CORRADE_WITH_UTILITY) target_compile_definitions(CorradeUtilityTestLib PRIVATE "CORRADE_GRACEFUL_ASSERT") if(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH) target_compile_definitions(CorradeUtilityTestLib PUBLIC "CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH") + if(CORRADE_TARGET_EMSCRIPTEN) + target_compile_options(CorradeUtilityTestLib PUBLIC "-msimd128") + endif() endif() set_target_properties(CorradeUtilityTestLib PROPERTIES DEBUG_POSTFIX "-d") if(CORRADE_BUILD_STATIC_PIC) From 96779d63d04982cf48fdc72ae97084573d7dc1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 28 Jul 2022 21:32:45 +0200 Subject: [PATCH 63/79] Containers: unroll < 16b fallback in StringView::find(). Doesn't seem to make the usual case any slower, but makes the small case 1.5-1.8x faster on x86 and 25% on WASM. Which is quite significant. Couldn't make a common helper to cover both because then it stopped being fast on WASM. Not on x86 tho. Strange, hitting some inlined size limit?? --- src/Corrade/Containers/StringView.cpp | 58 +++++++++++++++++++++------ 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index d86ab7469..306e96192 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -227,12 +227,28 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay const char* { const char* const end = data + size; - /* If we have less than 16 bytes, do it the stupid way */ - /** @todo SWAR?? */ - if(size < 16) { - for(const char* i = data; i != end; ++i) - if(*i == character) return i; - return {}; + /* If we have less than 16 bytes, do it the stupid way. Compared to a plain + loop this is 1.5-2x faster when unrolled. */ + { + const char* j = data - 1; + switch(size) { + case 15: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 14: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 13: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 12: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 11: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 10: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 9: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 8: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 7: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 6: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 5: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 4: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 3: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 2: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 1: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 0: return {}; + } } const __m128i vn1 = _mm_set1_epi8(character); @@ -397,12 +413,30 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay const char* { const char* const end = data + size; - /* If we have less than 16 bytes, do it the stupid way */ - /** @todo SWAR?? */ - if(size < 16) { - for(const char* i = data; i != end; ++i) - if(*i == character) return i; - return {}; + /* If we have less than 16 bytes, do it the stupid way. Compared to a plain + loop, this is 25% faster when unrolled. Strangely enough, if the switch + is put into an external always inline function to avoid duplication with + the SSE2 variant, it no longer gives the advantage. */ + { + const char* j = data - 1; + switch(size) { + case 15: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 14: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 13: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 12: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 11: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 10: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 9: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 8: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 7: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 6: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 5: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 4: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 3: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 2: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 1: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 0: return {}; + } } const v128_t vn1 = wasm_i8x16_splat(character); From a373ceffbe8d9786eacaed93923e96275eaa5f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 12:50:00 +0200 Subject: [PATCH 64/79] Containers: further optimize unrolled String::find(). --- src/Corrade/Containers/StringView.cpp | 56 ++++++++++++++++++--------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 306e96192..d85856456 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -228,25 +228,43 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay + repeated 15 times (with <+243> returning %r8 for all), while a + post-increment and `return j - 1` is just + lea 0x1(%rax),%rcx + cmp (%rax),%dl + je 0x63f20 <+208> + with %rax and %rcx alternating in every case and the jump always + different. That's 25% instructions less for the post-increment, and the + benchmark confirms that (~3.50 vs ~2.80 µs). Clang (13) does a similar + thing, although it has `lea, cmp, mov, je` in the first case instead and + `cmp, je, add` in the second case instead, and (probably due to the + different order?) the benchmark doesn't show any difference between the + two. Since post-increment significantly helps GCC and doesn't make + Clang slower, use it. */ { - const char* j = data - 1; + const char* j = data; switch(size) { - case 15: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 14: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 13: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 12: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 11: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 10: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 9: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 8: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 7: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 6: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 5: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 4: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 3: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 2: if(*++j == character) return j; CORRADE_FALLTHROUGH - case 1: if(*++j == character) return j; CORRADE_FALLTHROUGH + case 15: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 14: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 13: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 12: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 11: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 10: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 9: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 8: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 7: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 6: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 5: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 4: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 3: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 2: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH + case 1: if(*j++ == character) return j - 1; CORRADE_FALLTHROUGH case 0: return {}; } } @@ -416,7 +434,9 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay Date: Fri, 29 Jul 2022 17:33:27 +0200 Subject: [PATCH 65/79] Containers: add a perf TODO for WASM SIMD128. I ... didn't expect to hit such massive instruction portability issues so soon. But even with that it's still running circles around the standard memchr(). --- src/Corrade/Containers/StringView.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index d85856456..68aeb5f35 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -488,6 +488,20 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED typename std::decay Date: Fri, 29 Jul 2022 11:35:58 +0200 Subject: [PATCH 66/79] Cpu: test a WASM SIMD128 instruction that's only in the finalized set. There was quite a lot of renaming and shuffling around before then, so be sure to verify only the actually finalized variant and not something else. --- src/Corrade/Test/CpuTest.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index d2fb5ba8c..f527c128a 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -1483,19 +1483,24 @@ template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(NEON_FP16) int callInstructionFor #endif #ifdef CORRADE_ENABLE_SIMD128 template<> CORRADE_NEVER_INLINE CORRADE_ENABLE(SIMD128) int callInstructionFor() { - v128_t a = wasm_i32x4_make(-10, 20, -30, 40); - - /* All instructions SIMD128 */ + v128_t a = wasm_f32x4_make(5.47, 2.23, 7.62, 0.5); + + /* All instructions SIMD128. wasm_f32x4_ceil() is only available in the + finalized wasm intrinsics that's since Clang 13 which is used since + Emscripten 2.0.13, thus any older version should not have the + CORRADE_ENABLE_SIMD128 macro defined: + https://github.com/llvm/llvm-project/commit/502f54049d17f5a107f833596fb2c31297a99773 + https://github.com/emscripten-core/emscripten/commit/deab7783df407b260f46352ffad2a77ca8fb0a4c */ union { v128_t v; - int s[8]; + float s[8]; } b; - b.v = wasm_i32x4_abs(a); + b.v = wasm_f32x4_ceil(a); - CORRADE_COMPARE(b.s[0], 10); - CORRADE_COMPARE(b.s[1], 20); - CORRADE_COMPARE(b.s[2], 30); - CORRADE_COMPARE(b.s[3], 40); + CORRADE_COMPARE(b.s[0], 6.0); + CORRADE_COMPARE(b.s[1], 3.0); + CORRADE_COMPARE(b.s[2], 8.0); + CORRADE_COMPARE(b.s[3], 1.0); return b.s[0]; } #endif From 39a98cf04bef849cf3366fa3561d9250d97e6c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 14:01:37 +0200 Subject: [PATCH 67/79] Define CORRADE_TARGET_SIMD128 only on Clang 13+ and Emscripten 2.0.18+. While the earlier versions support a certain WIP variant of the intrinsics, it's not all, and some of these were renamed, opcodes renumbered etc. Doesn't really make sense to pretend they're usable in that case. And then emsdk DOES NOT MAKE IT ANY SIMPLER by bundling a prerelease of it, thus we *also* have to check emscripten version to be really sure that SIMD128 can be used. UGH --- src/Corrade/Corrade.h | 5 ++++- src/Corrade/configure.h.cmake | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Corrade/Corrade.h b/src/Corrade/Corrade.h index 893788ef0..b9f18d5a6 100644 --- a/src/Corrade/Corrade.h +++ b/src/Corrade/Corrade.h @@ -800,7 +800,10 @@ GCC/Clang). Superset of @ref CORRADE_TARGET_NEON_FMA. Defined on @ref CORRADE_TARGET_WASM "WebAssembly" if [128-bit SIMD](https://github.com/webassembly/simd) instructions are enabled at -compile time (`-msimd128` passed to Clang). +compile time (`-msimd128` passed to Clang), and the compiler supports the +finalized version of the intrinsics, which is since Clang 13 and Emscripten +2.0.18. Emscripten SDK 2.0.13 to 2.0.17 ship with a Clang that reports as 13 +but isn't actually the final version. @see @relativeref{Corrade,Cpu}, @relativeref{Corrade,Cpu::Simd128}, @ref CORRADE_ENABLE_SIMD128 */ diff --git a/src/Corrade/configure.h.cmake b/src/Corrade/configure.h.cmake index b33d4c728..a6e00d0d3 100644 --- a/src/Corrade/configure.h.cmake +++ b/src/Corrade/configure.h.cmake @@ -335,9 +335,16 @@ #endif #endif -/* Undocumented, checked via `echo | em++ -x c++ -dM -E - -msimd128` */ +/* Undocumented, checked via `echo | em++ -x c++ -dM -E - -msimd128`. + Restricting to the finalized SIMD variant, which is since Clang 13: + https://github.com/llvm/llvm-project/commit/502f54049d17f5a107f833596fb2c31297a99773 + Emscripten 2.0.13 sets Clang 13 as the minimum, however it doesn't imply + that emsdk 2.0.13 actually contains the final Clang 13. That's only since + 2.0.18, thus to avoid nasty issues we have to check Emscripten version as + well :( + https://github.com/emscripten-core/emscripten/commit/deab7783df407b260f46352ffad2a77ca8fb0a4c */ #elif defined(CORRADE_TARGET_WASM) -#ifdef __wasm_simd128__ +#if defined(__wasm_simd128__) && __clang_major__ >= 13 && __EMSCRIPTEN_major__*10000 + __EMSCRIPTEN_minor__*100 + __EMSCRIPTEN_tiny__ >= 20018 #define CORRADE_TARGET_SIMD128 #endif #endif From b826e5ed6aefc3ce150c886fbdaa190957f2b1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 18:04:27 +0200 Subject: [PATCH 68/79] doc: work around Emscripten 2.0.20 being extra picky. --- doc/snippets/CMakeLists.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/snippets/CMakeLists.txt b/doc/snippets/CMakeLists.txt index e3a0ac4c4..96a88e4f4 100644 --- a/doc/snippets/CMakeLists.txt +++ b/doc/snippets/CMakeLists.txt @@ -128,6 +128,22 @@ if(CORRADE_WITH_TESTSUITE AND NOT CORRADE_TARGET_IOS) target_link_libraries(testsuite-iteration PUBLIC CorradeTestSuite) target_link_libraries(testsuite-benchmark PUBLIC CorradeTestSuite) target_link_libraries(testsuite-benchmark-custom PUBLIC CorradeTestSuite) + + if(CORRADE_TARGET_EMSCRIPTEN) + # Wasn't a problem for Emscripten <= 2.0.13 and isn't a problem for + # 3.1+, but for some reason 2.0.20 needs this. The corrade_add_test() + # macro does this automatically, but here we need to do it manually. + set_property(TARGET + testsuite-basic + testsuite-templated + testsuite-repeated + testsuite-instanced + testsuite-save-diagnostic + testsuite-iteration + testsuite-benchmark + testsuite-benchmark-custom + APPEND_STRING PROPERTY LINK_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") + endif() endif() if(CORRADE_TARGET_ANDROID) From d15c2ea0a2ae509b1abc1c8169d37110853ce32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 19:12:34 +0200 Subject: [PATCH 69/79] modules: retrieve Node.js version in FindNodeJs. --- modules/FindNodeJs.cmake | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/FindNodeJs.cmake b/modules/FindNodeJs.cmake index 8823e6167..3169eb94b 100644 --- a/modules/FindNodeJs.cmake +++ b/modules/FindNodeJs.cmake @@ -6,6 +6,7 @@ # # NodeJs_FOUND - True if Node.js executable is found # NodeJs::NodeJs - Node.js executable imported target +# NodeJs_VERSION - Version string reported by ``node --version`` # # @@ -37,8 +38,21 @@ find_program(NODEJS_EXECUTABLE node) mark_as_advanced(NODEJS_EXECUTABLE) +if(NODEJS_EXECUTABLE) + execute_process(COMMAND ${NODEJS_EXECUTABLE} --version + OUTPUT_VARIABLE NodeJs_VERSION + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NodeJs_VERSION MATCHES "^v[0-9]") + string(SUBSTRING ${NodeJs_VERSION} 1 -1 NodeJs_VERSION) + else() + unset(NodeJs_VERSION) + endif() +endif() + include(FindPackageHandleStandardArgs) -find_package_handle_standard_args("NodeJs" DEFAULT_MSG NODEJS_EXECUTABLE) +find_package_handle_standard_args(NodeJs + REQUIRED_VARS NODEJS_EXECUTABLE + VERSION_VAR NodeJs_VERSION) if(NOT TARGET NodeJs::NodeJs) add_executable(NodeJs::NodeJs IMPORTED) From 69363964380b5cac50c591c9444cfed802cd8fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 19:20:39 +0200 Subject: [PATCH 70/79] CMake: pass --experimental-wasm-simd to Node.js < 17 in corrade_add_test(). Snce version 15 Node.js ships with V8 8.6, which contains also the remaining bitmask instructions, so from that version onwards there should be no harm from having SIMD enabled. Older versions have SIMD incomplete, meaning general code won't really run anyway, independently of the flag being set or not. --- modules/UseCorrade.cmake | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/UseCorrade.cmake b/modules/UseCorrade.cmake index 7093ede76..016104ea5 100644 --- a/modules/UseCorrade.cmake +++ b/modules/UseCorrade.cmake @@ -450,7 +450,22 @@ function(corrade_add_test test_name) set_property(TARGET ${test_name} APPEND_STRING PROPERTY COMPILE_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") set_property(TARGET ${test_name} APPEND_STRING PROPERTY LINK_FLAGS " -s DISABLE_EXCEPTION_CATCHING=0") find_package(NodeJs REQUIRED) - add_test(NAME ${test_name} COMMAND NodeJs::NodeJs $ ${arguments}) + # Node.js before version 17 needs --experimental-wasm-simd. Version + # 17 upgraded to V8 9.1, which has the option enabled by default: + # https://github.com/nodejs/node/commit/a7cbf19a82c75e9a65e90fb8ba4947e2fc52ef39 + # Since the code defining these flags explicitly says "remove once + # they hit stable": + # https://github.com/v8/v8/blob/ba8ad5dd17ea85c856c09c2ff603641487d1f0ca/src/wasm/wasm-feature-flags.h#L101-L109 + # even though it causes numberous issues such as: + # https://github.com/nodejs/node/issues/43592 + # I'm future-proofing and not setting it for version 17+. From the + # other side, the flag goes back to ancient version 6 (2017), so it + # should be no problem to just pass it there always: + # https://github.com/v8/v8/commit/45618a9ab5cc98a5de200b0116670e8c272f0c5f + if(NodeJS_VERSION VERSION_LESS 17) + set(experimental_wasm_simd --experimental-wasm-simd) + endif() + add_test(NAME ${test_name} COMMAND NodeJs::NodeJs ${experimental_wasm_simd} $ ${arguments}) # Embed all files foreach(file ${files}) From f7f6fac8cccbabd4e6273b78f154921e03c5213e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 20:32:34 +0200 Subject: [PATCH 71/79] CMake: add our module path earlier. Need to use NodeJs_VERSION in order to figure out a default for a CMake option. The curse runs deep, yes? --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f1050436..514b07522 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,8 @@ project(Corrade CXX) # Use folders for nice tree in Visual Studio and XCode set_property(GLOBAL PROPERTY USE_FOLDERS ON) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/modules/" ${CMAKE_MODULE_PATH}) + include(CMakeDependentOption) # Options that used to be unprefixed. New options shouldn't be added to this @@ -331,7 +333,6 @@ if(CORRADE_BUILD_TESTS) enable_testing() endif() -set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/modules/" ${CMAKE_MODULE_PATH}) if(CMAKE_CROSSCOMPILING) find_program(CORRADE_RC_EXECUTABLE corrade-rc) if(NOT CORRADE_RC_EXECUTABLE) From d218e7805061348d3067151d934387389cf12482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 20:35:18 +0200 Subject: [PATCH 72/79] CMake: introduce CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128. I... can't even. This was supposed to be a simple one-line change, not an investigation spanning several days!! --- CMakeLists.txt | 31 ++++++++++++++++++++++++++++++ doc/building-corrade.dox | 10 ++++++++++ src/Corrade/Utility/CMakeLists.txt | 2 +- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 514b07522..0c7d2efd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,6 +264,37 @@ option(CORRADE_BUILD_CPU_RUNTIME_DISPATCH "Build with runtime dispatch for CPU-d # this for benchmarks. cmake_dependent_option(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH "Force pointer-based dispatch for unit tests to verify all variants of CPU-dependent functionality" ON "CORRADE_BUILD_TESTS" OFF) +# Pass -msimd128 to tests to make it possible to verify also the WASM SIMD +# functionality even if the actual library is built without. I thought it would +# be a no-brainer, however *actual* finalized SIMD128 requires final Clang 13 +# (such version is reported since emsdk 2.0.13, but the actual final version +# containing all intrinsics is only in emsdk 2.0.18), and Node.js 15+, which +# contains V8 8.6: +# https://github.com/nodejs/node/blob/main/doc/changelogs/CHANGELOG_V15.md#2020-10-20-version-1500-current-bethgriggs +# Final bitmask instructions were added in 8.5, Node.js 14 has 8.4: +# https://github.com/v8/v8/commit/aa5bcc09bf7d5d77056e6033918d79c30aa3f49a +# However, even now, in the midst of 2022 wildfires, emsdk still bundles +# Node.js 14.18 and there's no hope of it being upgraded any time soon: +# https://github.com/emscripten-core/emsdk/pull/877 +# https://github.com/emscripten-core/emsdk/issues/947 +# https://github.com/emscripten-core/emsdk/issues/1064 +# So instead of having -msimd128 always, I only enable it implicitly if Node.js +# 15 is found, and then CORRADE_ENABLE_SIMD128 / CORRADE_TARGET_SIMD128 gets +# enabled only if there's Clang 13 but also Emscripten 2.0.18+. What a mess. +if(CORRADE_TARGET_EMSCRIPTEN) + set(ENABLE_IF_NODEJS_15 OFF) + if(CORRADE_BUILD_TESTS) + find_package(NodeJs REQUIRED QUIET) + if(NOT NodeJs_VERSION VERSION_LESS 15.0) + set(ENABLE_IF_NODEJS_15 ON) + elseif(NOT DEFINED CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128) + message(WARNING "Disabling CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128 by default as finalized WebAssembly SIMD support is only since Node.js 15 but found version ${NodeJs_VERSION}") + endif() + endif() + + cmake_dependent_option(CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128 "Force -msimd128 for unit tests to verify also WASM SIMD variants of CPU-dependent functionality" ${ENABLE_IF_NODEJS_15} "CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH" OFF) +endif() + # Backwards compatibility for unprefixed CMake options. If the user isn't # explicitly using prefixed options in the first run already, accept the # unprefixed options, and remember this decision for subsequent runs diff --git a/doc/building-corrade.dox b/doc/building-corrade.dox index 1aa463e52..bbc74fa09 100644 --- a/doc/building-corrade.dox +++ b/doc/building-corrade.dox @@ -652,6 +652,16 @@ Building of tests is controlled by the following options: overhead from function pointers may however be significant on certain platforms, skewing benchmark results. Disable it to make the tests use the same dispatch method as the rest of the library. +- `CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128` --- Force Emscripten unit tests to + be built with `-msimd128` independenently of `CMAKE_CXX_FLAGS`. Together + with `CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH` and presence of the + @ref CORRADE_ENABLE_SIMD128 preprocessor variable this makes it possible + for the tests to verify also WebAssembly SIMD variants of CPU-dependent + functionality instead of just the scalar version if the rest of the library + is built without `-msimd128`. Enabled by default if + `CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH` is set and Node.js version + 15+ is found, which has finalized WebAssembly SIMD support. Disable to make + the tests use the same compilation flags as the rest of the library. The tests use Corrade's own @relativeref{Corrade,TestSuite} framework and can be run either manually (the binaries are located in `Test/` subdirectories in diff --git a/src/Corrade/Utility/CMakeLists.txt b/src/Corrade/Utility/CMakeLists.txt index 5036ad2e2..1d6762a15 100644 --- a/src/Corrade/Utility/CMakeLists.txt +++ b/src/Corrade/Utility/CMakeLists.txt @@ -230,7 +230,7 @@ if(CORRADE_WITH_UTILITY) target_compile_definitions(CorradeUtilityTestLib PRIVATE "CORRADE_GRACEFUL_ASSERT") if(CORRADE_BUILD_TESTS_FORCE_CPU_POINTER_DISPATCH) target_compile_definitions(CorradeUtilityTestLib PUBLIC "CORRADE_UTILITY_FORCE_CPU_POINTER_DISPATCH") - if(CORRADE_TARGET_EMSCRIPTEN) + if(CORRADE_BUILD_TESTS_FORCE_WASM_SIMD128) target_compile_options(CorradeUtilityTestLib PUBLIC "-msimd128") endif() endif() From 6dcf752746a636de2cb828aaedc65817e5633870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Fri, 29 Jul 2022 20:51:18 +0200 Subject: [PATCH 73/79] package/ci: I tried EMSDK and SIMD. Wasted whole day. Failed. --- doc/building-corrade.dox | 5 +++-- package/ci/circleci.yml | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/building-corrade.dox b/doc/building-corrade.dox index bbc74fa09..136007651 100644 --- a/doc/building-corrade.dox +++ b/doc/building-corrade.dox @@ -986,8 +986,9 @@ for more information. @subsection building-corrade-ci-circleci CircleCI In `package/ci/` there is a `circle.yml` file with Linux GCC 4.8, Linux ARM64, -macOS, Emscripten, AddressSanitizer, ThreadSanitizer, Android x86 and iOS -x86_64 configuration. Online at https://circleci.com/gh/mosra/corrade. +macOS, Emscripten (except SIMD, see the file for details), AddressSanitizer, +ThreadSanitizer, Android x86 and iOS x86_64 configuration. Online at +https://circleci.com/gh/mosra/corrade. @subsection building-corrade-ci-appveyor AppVeyor diff --git a/package/ci/circleci.yml b/package/ci/circleci.yml index 37511ab58..8c7607962 100644 --- a/package/ci/circleci.yml +++ b/package/ci/circleci.yml @@ -19,6 +19,14 @@ executors: # filesystem race and randomly complains that a file doesn't exist: # https://github.com/mosra/magnum/issues/413, # https://github.com/emscripten-core/emscripten/pull/10161 + # + # Regarding SIMD, while 1.39.6 supports some WIP variant of it, the + # finalized support is only since 2.0.18 (and then the first non-broken + # emsdk is 2.0.25). But emsdk ships with Node.js 14, and only Node.js 15 + # supports intrinsics that Emscripten 2.0.18 can produce, so there's no + # point in even trying, we wouldn't be able to run the tests anyway. + # TODO: revisit when https://github.com/emscripten-core/emsdk/issues/1064 + # or any other referenced issues are finally resolved - image: emscripten/emsdk:1.39.6-upstream python-3_6: docker: From 7a7a87213a08360e1536c16d34b10b8a18e09782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 30 Jul 2022 13:21:38 +0200 Subject: [PATCH 74/79] Cpu: mention an important caveat with CORRADE_ENABLE(). It surely sounds like there's a lot more buts than actually working features. --- src/Corrade/Cpu.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index d4c009f8e..a7d27de95 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -2998,6 +2998,11 @@ MSVC it's just a shorthand for multiple `CORRADE_ENABLE_*` macros one after another. On MSVC expands to nothing --- there the functions aren't annotated in anyway and moreover the default preprocessor behavior would make this extremely tricky to implement. + +@attention Due to the way the attributes are combined on Clang < 8 and GCC, in + certain cases the macro may silently accept even arguments that don't have + a corresponding `CORRADE_ENABLE_*` macro defined. To prevent portability + issues, pay extra attention to have a matching @cpp #ifdef @ce guard. */ #if !defined(CORRADE_TARGET_MSVC) || defined(CORRADE_TARGET_CLANG_CL) #define CORRADE_ENABLE(...) _CORRADE_HELPER_PICK(__VA_ARGS__, _CORRADE_ENABLE8, _CORRADE_ENABLE7, _CORRADE_ENABLE6, _CORRADE_ENABLE5, _CORRADE_ENABLE4, _CORRADE_ENABLE3, _CORRADE_ENABLE2, _CORRADE_ENABLE1, )(__VA_ARGS__) From ac375feaf74c75853a575759af6c3ea593704692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 30 Jul 2022 13:23:31 +0200 Subject: [PATCH 75/79] Cpu: rethink how GCC 9.3 lambda attribute bug is handled. Since 9.3 is the stock default on 20.04, it doesn't sound practical to just fail compilation there. I can't tell everyone to apt install g++-9 and then suffer through compiler switching in all their pipelines and CI scripts. Another option I considered was disabling (not defining) *all* CORRADE_ENABLE_* macros on this compiler, but again that would mean Ubuntu 20.04 users would silently have a much worse perf unless they switch compilers or use -march=native. Also far from ideal. Instead I'm just not using trailing return types anymore, which is actually the simplest way to get rid of this problem. Also adding a dedicated test for CORRADE_ENABLE_* and lambdas into CpuTest, since this was so far only tested in production, i.e. inside StringView code. Which is no good. This reverts commit 6615e449b793fa0f7b1d1db5a1318d916627bac5. --- package/ci/circleci.yml | 8 +- src/Corrade/Containers/StringView.cpp | 14 ++- src/Corrade/Cpu.h | 13 ++- src/Corrade/Test/CpuTest.cpp | 159 ++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 15 deletions(-) diff --git a/package/ci/circleci.yml b/package/ci/circleci.yml index 8c7607962..a2c07963b 100644 --- a/package/ci/circleci.yml +++ b/package/ci/circleci.yml @@ -97,14 +97,10 @@ commands: background: true - run: name: Update apt and install base packages - # Compared to Docker images this needs sudo. Default gcc is 9.3, which - # has https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 and breaks our - # CPU dispatch, 9.4 is for some reason a separate package. + # Compared to Docker images this needs sudo command: | sudo apt update - sudo apt install -y ninja-build g++-9 - echo 'export CC=gcc-9' >> $BASH_ENV - echo 'export CXX=g++-9' >> $BASH_ENV + sudo apt install -y ninja-build install-gcc-4_8: steps: diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 68aeb5f35..ba27b60c5 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -224,7 +224,9 @@ CORRADE_ENABLE(SSE2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleV } CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Sse2|Cpu::Bmi1)) { - return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(SSE2,BMI1) -> const char* { + /* Can't use trailing return type due to a GCC 9.3 bug, which is the default + on Ubuntu 20.04: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(SSE2,BMI1) { const char* const end = data + size; /* If we have less than 16 bytes, do it the stupid way. Compared to a plain @@ -265,7 +267,7 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay(nullptr); } } @@ -324,7 +326,7 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(SSE2,BMI1) typename std::decay(nullptr); }; } #endif @@ -348,7 +350,9 @@ CORRADE_ENABLE(AVX2,BMI1) CORRADE_ALWAYS_INLINE const char* findCharacterSingleV } CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Avx2|Cpu::Bmi1)) { - return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(AVX2,BMI1) -> const char* { + /* Can't use trailing return type due to a GCC 9.3 bug, which is the default + on Ubuntu 20.04: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(AVX2,BMI1) { const char* const end = data + size; /* If we have less than 32 bytes, fall back to the SSE variant */ @@ -412,7 +416,7 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay(nullptr); }; } #endif diff --git a/src/Corrade/Cpu.h b/src/Corrade/Cpu.h index a7d27de95..dfd86ac53 100644 --- a/src/Corrade/Cpu.h +++ b/src/Corrade/Cpu.h @@ -306,16 +306,21 @@ actual arguments: @m_class{m-block m-danger } -@par Instruction enabling macros +@par Instruction enabling macros and lambdas An important difference with the @ref CORRADE_ENABLE_SSE2 "CORRADE_ENABLE_*" macros is that they now have to go also directly next to the lambda as GCC [currently doesn't propagate the attributes](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80439) from the wrapper function to the nested lambda. To make matters worse, older versions of Clang suffer from the inverse problem and ignore lambda attributes, so you have to specify them on both the lambda and the wrapper - function. GCC 9.1 to 9.3 also has a [bug where it can't parse lambda attributes](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333), - use version 8, 9.4 or 10.3 instead. The above AVX variant would look like - this with relevant macros added: + function. GCC 9.1 to 9.3 also has a [bug where it can't parse attributes on lambdas with a trailing return type](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333). + The preferrable solution is to not use a trailing return type at the cost + of potentially more verbose @cpp return @ce statements. Alternatively you + can require version 8, 9.4 or 10 instead, but note that 9.3 is the default + compiler on Ubuntu 20.04. +@par + All things considered, the above AVX variant would look like this with + relevant macros added: @par @snippet Corrade.cpp Cpu-usage-automatic-runtime-dispatch-target-attributes diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index f527c128a..dedce77d5 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -110,6 +110,9 @@ struct CpuTest: TestSuite::Tester { void enableMacrosMultiple(); void enableMacrosMultipleAllEmpty(); + void enableMacrosLambda(); + void enableMacrosLambdaMultiple(); + void debug(); void debugPacked(); }; @@ -191,6 +194,9 @@ CpuTest::CpuTest() { &CpuTest::enableMacrosMultiple, &CpuTest::enableMacrosMultipleAllEmpty, + &CpuTest::enableMacrosLambda, + &CpuTest::enableMacrosLambdaMultiple, + &CpuTest::debug, &CpuTest::debugPacked}); } @@ -1599,6 +1605,159 @@ void CpuTest::enableMacrosMultipleAllEmpty() { CORRADE_COMPARE(callInstructionMultipleAllEmpty(), 1); } +/* On Clang it's enough to have the ENABLE macro just on the wrapper function. + On GCC it has to be attached to the lambda due to + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80439. However, older versions + of Clang suffer from the inverse problem and ignore lambda attributes, so it + has to stay on the function definition as well. Furthermore, if the trailing + return type is uncommented, the code will fail to compile on GCC 9.1 to 9.3: + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ +#ifdef CORRADE_ENABLE_AVX2 +CORRADE_ENABLE_AVX2 int callInstructionLambda() { + return []() CORRADE_ENABLE_AVX2 /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Avx2)) + CORRADE_SKIP("AVX2 feature not supported"); + + /* Same as callInstructionFor() */ + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* If the AVX2 instructions aren't enabled, this will fail to link */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00); + return mask; + }(); +} +#elif defined(CORRADE_ENABLE_NEON) +CORRADE_ENABLE_NEON int callInstructionLambda() { + return []() CORRADE_ENABLE_NEON /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Neon)) + CORRADE_SKIP("NEON feature not supported"); + + /* Same as callInstructionFor() */ + int32x4_t a{-10, 20, -30, 40}; + + union { + int32x4_t v; + int s[8]; + } b; + b.v = vabsq_s32(a); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; + }(); +} +#elif defined(CORRADE_ENABLE_SIMD128) +CORRADE_ENABLE_SIMD128 int callInstructionLambda() { + return []() CORRADE_ENABLE_SIMD128 /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Simd128)) + CORRADE_SKIP("SIMD128 feature not supported"); + + /* Same as callInstructionFor() */ + v128_t a = wasm_f32x4_make(5.47, 2.23, 7.62, 0.5); + + union { + v128_t v; + float s[8]; + } b; + b.v = wasm_f32x4_ceil(a); + + CORRADE_COMPARE(b.s[0], 6.0); + CORRADE_COMPARE(b.s[1], 3.0); + CORRADE_COMPARE(b.s[2], 8.0); + CORRADE_COMPARE(b.s[3], 1.0); + return b.s[0]; + }(); +} +#else +int callInstructionLambda() { + CORRADE_SKIP("No usable CORRADE_ENABLE_ macros defined"); +} +#endif + +void CpuTest::enableMacrosLambda() { + /* Verifies that CORRADE_ENABLE_* can be applied also to lambdas. See the + comment above the implementations for more information. */ + CORRADE_VERIFY(callInstructionLambda()); +} + +/* Same as callInstructionLambda(), just with CORRADE_ENABLE(*) instead of + CORRADE_ENABLE_* */ +#ifdef CORRADE_ENABLE_AVX2 +CORRADE_ENABLE(AVX2) int callInstructionLambdaMultiple() { + return []() CORRADE_ENABLE(AVX2) /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Avx2)) + CORRADE_SKIP("AVX2 feature not supported"); + + /* Same as callInstructionFor() */ + __m256i a = _mm256_set_epi64x(0x8080808080808080ull, 0, 0x8080808080808080ull, 0); + + /* If the AVX2 instructions aren't enabled, this will fail to link */ + int mask = _mm256_movemask_epi8(a); + + CORRADE_COMPARE(mask, 0xff00ff00); + return mask; + }(); +} +#elif defined(CORRADE_ENABLE_NEON) +CORRADE_ENABLE(NEON) int callInstructionLambdaMultiple() { + return []() CORRADE_ENABLE(NEON) /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Neon)) + CORRADE_SKIP("NEON feature not supported"); + + /* Same as callInstructionFor() */ + int32x4_t a{-10, 20, -30, 40}; + + union { + int32x4_t v; + int s[8]; + } b; + b.v = vabsq_s32(a); + + CORRADE_COMPARE(b.s[0], 10); + CORRADE_COMPARE(b.s[1], 20); + CORRADE_COMPARE(b.s[2], 30); + CORRADE_COMPARE(b.s[3], 40); + return b.s[0]; + }(); +} +#elif defined(CORRADE_ENABLE_SIMD128) +CORRADE_ENABLE(SIMD128) int callInstructionLambdaMultiple() { + return []() CORRADE_ENABLE(SIMD128) /*-> int*/ { + if(!(Cpu::runtimeFeatures() & Cpu::Simd128)) + CORRADE_SKIP("SIMD128 feature not supported"); + + /* Same as callInstructionFor() */ + v128_t a = wasm_f32x4_make(5.47, 2.23, 7.62, 0.5); + + union { + v128_t v; + float s[8]; + } b; + b.v = wasm_f32x4_ceil(a); + + CORRADE_COMPARE(b.s[0], 6.0); + CORRADE_COMPARE(b.s[1], 3.0); + CORRADE_COMPARE(b.s[2], 8.0); + CORRADE_COMPARE(b.s[3], 1.0); + return b.s[0]; + }(); +} +#else +int callInstructionLambdaMultiple() { + CORRADE_SKIP("No usable CORRADE_ENABLE_ macros defined"); +} +#endif + +void CpuTest::enableMacrosLambdaMultiple() { + /* Verifies that CORRADE_ENABLE(*) (the function macro variant) can be + applied to lambdas as well */ + CORRADE_VERIFY(callInstructionLambdaMultiple()); +} + void CpuTest::debug() { /* If more flags get added, this might need to become even more zeros */ const unsigned int dead = 0xde00ad00; From b6552f9626511d5c0efe041cf26788d9cff8c1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 30 Jul 2022 13:46:44 +0200 Subject: [PATCH 76/79] Containers: add a perf TODO. --- src/Corrade/Containers/StringView.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index ba27b60c5..27f4609ef 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -391,6 +391,9 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay Date: Sun, 31 Jul 2022 21:35:49 +0200 Subject: [PATCH 77/79] Cpu: add a benchmark for dispatch overhead. Comparing regular functions, function pointers, IFUNCs and all that also in an external (dynamic) library. --- src/Corrade/Test/CMakeLists.txt | 11 +- src/Corrade/Test/CpuTest.cpp | 123 +++++++++++++++++++- src/Corrade/Test/CpuTestExternalLibrary.cpp | 55 +++++++++ src/Corrade/Test/CpuTestExternalLibrary.h | 52 +++++++++ 4 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 src/Corrade/Test/CpuTestExternalLibrary.cpp create mode 100644 src/Corrade/Test/CpuTestExternalLibrary.h diff --git a/src/Corrade/Test/CMakeLists.txt b/src/Corrade/Test/CMakeLists.txt index 58d69f368..4df49c845 100644 --- a/src/Corrade/Test/CMakeLists.txt +++ b/src/Corrade/Test/CMakeLists.txt @@ -33,7 +33,16 @@ corrade_add_test(MainTest MainTest.cpp # Prefixed with project name to avoid conflicts with TagsTest in Magnum corrade_add_test(CorradeTagsTest TagsTest.cpp) -corrade_add_test(CpuTest CpuTest.cpp) +# Platforms like Emscripten or Android that don't have dynamic plugin support +# don't really have dynamic libraries either. Thus there's no point in trying +# to benchmark a call into an external dynamic library. +if(CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT) + add_library(CpuTestExternalLibrary STATIC CpuTestExternalLibrary.cpp) +else() + add_library(CpuTestExternalLibrary SHARED CpuTestExternalLibrary.cpp) +endif() +target_link_libraries(CpuTestExternalLibrary PRIVATE CorradeUtility) +corrade_add_test(CpuTest CpuTest.cpp LIBRARIES CpuTestExternalLibrary) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/configure.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/configure.h) diff --git a/src/Corrade/Test/CpuTest.cpp b/src/Corrade/Test/CpuTest.cpp index dedce77d5..bcb314441 100644 --- a/src/Corrade/Test/CpuTest.cpp +++ b/src/Corrade/Test/CpuTest.cpp @@ -53,6 +53,8 @@ #include #endif +#include "CpuTestExternalLibrary.h" + namespace Corrade { namespace Test { namespace { struct CpuTest: TestSuite::Tester { @@ -105,6 +107,14 @@ struct CpuTest: TestSuite::Tester { void tagDispatchedPointer(); void tagDispatchedIfunc(); + void benchmarkTagDispatchedCompileTime(); + void benchmarkTagDispatchedPointer(); + void benchmarkTagDispatchedIfunc(); + void benchmarkTagDispatchedExternalLibraryCompileTime(); + void benchmarkTagDispatchedExternalLibraryPointer(); + void benchmarkTagDispatchedExternalLibraryIfunc(); + void benchmarkTagDispatchedExternalLibraryEveryCall(); + template void enableMacros(); void enableMacrosMultiple(); @@ -167,8 +177,17 @@ CpuTest::CpuTest() { &CpuTest::tagDispatchedCompileTime, &CpuTest::tagDispatchedPointer, - &CpuTest::tagDispatchedIfunc, + &CpuTest::tagDispatchedIfunc}); + addBenchmarks({&CpuTest::benchmarkTagDispatchedCompileTime, + &CpuTest::benchmarkTagDispatchedPointer, + &CpuTest::benchmarkTagDispatchedIfunc, + &CpuTest::benchmarkTagDispatchedExternalLibraryCompileTime, + &CpuTest::benchmarkTagDispatchedExternalLibraryPointer, + &CpuTest::benchmarkTagDispatchedExternalLibraryIfunc, + &CpuTest::benchmarkTagDispatchedExternalLibraryEveryCall}, 100); + + addTests({ #ifdef CORRADE_TARGET_X86 &CpuTest::enableMacros, &CpuTest::enableMacros, @@ -1203,6 +1222,108 @@ void CpuTest::tagDispatchedIfunc() { #endif } +constexpr std::size_t BenchmarkDispatchedRepeats = 1000000; + +/* Otherwise Clang inlines even through the function pointer */ +#ifdef CORRADE_TARGET_CLANG +CORRADE_NEVER_INLINE +#endif +int benchmarkDispatchedImplementation(int a) { + return a + 1; +} + +CORRADE_NEVER_INLINE int benchmarkDispatchedCompileTime(int a) { + /* Because benchmarkDispatchedImplementation() is marked as never inline + on Clang, calling it from here would result in two deinlined calls, + skewing the benchmark */ + #ifdef CORRADE_TARGET_CLANG + return a + 1; + #else + return benchmarkDispatchedImplementation(a); + #endif +} + +void CpuTest::benchmarkTagDispatchedCompileTime() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedCompileTime(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +auto benchmarkDispatchedImplementation(Cpu::Features) -> int(*)(int) { + return benchmarkDispatchedImplementation; +} + +CORRADE_CPU_DISPATCHED_POINTER(benchmarkDispatchedImplementation, int(*benchmarkDispatchedPointer)(int)) + +void CpuTest::benchmarkTagDispatchedPointer() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedPointer(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_DISPATCHED_IFUNC(benchmarkDispatchedImplementation, int benchmarkDispatchedIfunc(int)) +#endif + +void CpuTest::benchmarkTagDispatchedIfunc() { + #ifndef CORRADE_CPU_USE_IFUNC + CORRADE_SKIP("CORRADE_CPU_USE_IFUNC not available"); + #else + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedIfunc(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); + #endif +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryCompileTime() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryCompileTime(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryPointer() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryPointer(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryIfunc() { + #ifndef CORRADE_CPU_USE_IFUNC + CORRADE_SKIP("CORRADE_CPU_USE_IFUNC not available"); + #else + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryIfunc(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); + #endif +} + +void CpuTest::benchmarkTagDispatchedExternalLibraryEveryCall() { + int a = 0; + CORRADE_BENCHMARK(BenchmarkDispatchedRepeats) { + a = Test::benchmarkDispatchedExternalLibraryEveryCall(Cpu::compiledFeatures())(a); + } + + CORRADE_COMPARE(a, BenchmarkDispatchedRepeats); +} + /* Not using an argument here since we *don't* want the overload delegating in this case -- it would hide errors when a certain instruction set doesn't have a corresponding overload, as it'd fall back to a parent one. I'm also diff --git a/src/Corrade/Test/CpuTestExternalLibrary.cpp b/src/Corrade/Test/CpuTestExternalLibrary.cpp new file mode 100644 index 000000000..8a3f5fe76 --- /dev/null +++ b/src/Corrade/Test/CpuTestExternalLibrary.cpp @@ -0,0 +1,55 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "CpuTestExternalLibrary.h" + +#include "Corrade/Cpu.h" + +namespace Corrade { namespace Test { + +namespace { + +CORRADE_ALWAYS_INLINE auto implementation(Cpu::Features) -> int(*)(int) { + return [](int a) { return a + 1; }; +} + +} + +int benchmarkDispatchedExternalLibraryCompileTime(int a) { + return implementation(Cpu::DefaultBase)(a); +} + +CORRADE_CPU_DISPATCHED_POINTER(implementation, int (*benchmarkDispatchedExternalLibraryPointer)(int)) + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_DISPATCHED_IFUNC(implementation, int benchmarkDispatchedExternalLibraryIfunc(int)) +#endif + +auto benchmarkDispatchedExternalLibraryEveryCall(Cpu::Features features) -> int(*)(int) { + return implementation(features); +} + +}} diff --git a/src/Corrade/Test/CpuTestExternalLibrary.h b/src/Corrade/Test/CpuTestExternalLibrary.h new file mode 100644 index 000000000..7ebad2d9b --- /dev/null +++ b/src/Corrade/Test/CpuTestExternalLibrary.h @@ -0,0 +1,52 @@ +/* + This file is part of Corrade. + + Copyright © 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, + 2017, 2018, 2019, 2020, 2021, 2022 + Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include "Corrade/Corrade.h" +#include "Corrade/Utility/VisibilityMacros.h" + +#ifndef CORRADE_PLUGINMANAGER_NO_DYNAMIC_PLUGIN_SUPPORT + #ifdef CpuTestExternalLibrary_EXPORTS + #define CORRADE_CPU_TEST_EXPORT CORRADE_VISIBILITY_EXPORT + #else + #define CORRADE_CPU_TEST_EXPORT CORRADE_VISIBILITY_IMPORT + #endif +#else + #define CORRADE_CPU_TEST_EXPORT CORRADE_VISIBILITY_STATIC +#endif + +namespace Corrade { namespace Test { + +CORRADE_CPU_TEST_EXPORT int benchmarkDispatchedExternalLibraryCompileTime(int); + +CORRADE_CPU_TEST_EXPORT extern int(*benchmarkDispatchedExternalLibraryPointer)(int); + +#ifdef CORRADE_CPU_USE_IFUNC +CORRADE_CPU_TEST_EXPORT int benchmarkDispatchedExternalLibraryIfunc(int); +#endif + +CORRADE_CPU_TEST_EXPORT auto benchmarkDispatchedExternalLibraryEveryCall(Cpu::Features) -> int(*)(int); + +}} From f0f2569efcac4aa6ec6c6b6ceccf7fe332e9a19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sat, 30 Jul 2022 19:10:27 +0200 Subject: [PATCH 78/79] Containers: ARM NEON variant of String::find(). This was a *rewarning* learning experience. --- src/Corrade/Containers/StringView.cpp | 136 ++++++++++++++++++ .../Containers/Test/StringViewBenchmark.cpp | 6 + .../Containers/Test/StringViewTest.cpp | 3 + 3 files changed, 145 insertions(+) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 27f4609ef..2957d3c93 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -43,6 +43,9 @@ #if (defined(CORRADE_ENABLE_SSE2) || defined(CORRADE_ENABLE_AVX)) && defined(CORRADE_ENABLE_BMI1) #include "Corrade/Utility/IntrinsicsAvx.h" /* TZCNT is in AVX headers :( */ #endif +#ifdef CORRADE_ENABLE_NEON +#include +#endif #ifdef CORRADE_ENABLE_SIMD128 #include #endif @@ -202,6 +205,10 @@ namespace { variant --- effectively performing the lookup with either two overlapping 16-byte vectors (or falling back to scalar for less than 16 bytes). + The ARM variant has the high-level concept similar to x86, except that NEON + doesn't have a bitmask instruction. Instead a "right shift and narrow" + instruction is used, see comments there for details. + The WASM variant is mostly a direct translation of the x86 variant, except as noted in code comments. */ @@ -424,6 +431,135 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay(at)); + + /* Emulating _mm_movemask_epi8() on ARM is rather expensive, even the most + optimized variant listed at https://github.com/WebAssembly/simd/pull/201 + is 6+ instructions. Instead, a "shift right and narrow" is used, based + on an idea from https://twitter.com/Danlark1/status/1539344281336422400 + and further explained in https://github.com/facebook/zstd/pull/3139. + + First, similarly to x86, an equivalence mask is calculated with bytes + being either ff or 00 based on whether they match: + + 00 ff ff 00 00 00 ff ff 00 00 00 00 ff 00 00 00 + + The result is reinterpreted as 8 16bit values: + + 00ff ff00 0000 ffff 0000 0000 ff00 0000 + + Then, the vshrn_n_u16() instruction shifts each 16bit value four bits to + the right, and drops the high half: + + 000f 0ff0 0000 0fff 0000 0000 0ff0 0000 + 0f f0 00 ff 00 00 f0 00 + + The result, stored in the lower half of a 128-bit register, is then + extracted as a single 64-bit number: + + 0ff0 00ff 0000 f000 + + This effectively reduces the original 128-bit mask to a half, with every + four bits describing a masked byte. While that's still 4x more than what + _mm_movemask_epi8() produces, it can be tested against zero using + regular scalar operations. Finally, `__builtin_ctzll(mask) >> 2` is + equivalent to what TZCNT on a 16bit mask produced by _mm_movemask_epi8() + would return -- there's simply just 4x more bits. */ + const uint16x8_t eq16 = vreinterpretq_u16_u8(vceqq_u8(chunk, vn1)); + const uint64x1_t shrn64 = vreinterpret_u64_u8(vshrn_n_u16(eq16, 4)); + if(const uint64_t mask = vget_lane_u64(shrn64, 0)) + return at + (__builtin_ctzll(mask) >> 2); + return {}; +} + +CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(NEON) typename std::decay::type stringFindCharacterImplementation(CORRADE_CPU_DECLARE(Cpu::Neon)) { + /* Can't use trailing return type due to a GCC 9.3 bug, which is the default + on Ubuntu 20.04: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90333 */ + return [](const char* const data, const std::size_t size, const char character) CORRADE_ENABLE(NEON) { + const char* const end = data + size; + + /* If we have less than 16 bytes, do it the stupid way. Unlike x86 or WASM, + unrolling the loop here makes things actually worse. */ + /** @todo investigate why */ + if(size < 16) { + for(const char* i = data; i != end; ++i) + if(*i == character) return i; + return static_cast(nullptr); + } + + const uint8x16_t vn1 = vdupq_n_u8(character); + + /* Unconditionally do a lookup in the first vector a slower, unaligned + way. Any extra branching to avoid the unaligned load if already aligned + would be most probably more expensive than the actual unaligned load. */ + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Neon, data, vn1)) + return found; + + /* Go to the next aligned position. If the pointer was already aligned, + we'll go to the next aligned vector; if not, there will be an overlap + and we'll check some bytes twice. */ + const char* i = reinterpret_cast(reinterpret_cast(data + 16) & ~0xf); + CORRADE_INTERNAL_DEBUG_ASSERT(i >= data && reinterpret_cast(i) % 16 == 0); + + /* Go four vectors at a time with the aligned pointer */ + for(; i + 4*16 < end; i += 4*16) { + const uint8x16_t a = vld1q_u8(reinterpret_cast(i) + 0*16); + const uint8x16_t b = vld1q_u8(reinterpret_cast(i) + 1*16); + const uint8x16_t c = vld1q_u8(reinterpret_cast(i) + 2*16); + const uint8x16_t d = vld1q_u8(reinterpret_cast(i) + 3*16); + + const uint8x16_t eqa = vceqq_u8(vn1, a); + const uint8x16_t eqb = vceqq_u8(vn1, b); + const uint8x16_t eqc = vceqq_u8(vn1, c); + const uint8x16_t eqd = vceqq_u8(vn1, d); + + /* Similar to findCharacterSingleVectorUnaligned(Cpu::NeonT), except + that four "shift right and narrow" operations are done, interleaving + the result into two registers instead of four */ + const uint8x8_t maska = vshrn_n_u16(vreinterpretq_u16_u8(eqa), 4); + const uint8x16_t maskab = vshrn_high_n_u16(maska, vreinterpretq_u16_u8(eqb), 4); + const uint8x8_t maskc = vshrn_n_u16(vreinterpretq_u16_u8(eqc), 4); + const uint8x16_t maskcd = vshrn_high_n_u16(maskc, vreinterpretq_u16_u8(eqd), 4); + + /* Which makes it possible to test with just one OR and a horizontal + add instead of three ORs and a horizontal add */ + if(vaddvq_u8(vorrq_u8(maskab, maskcd))) { + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskab), 0)) + return i + 0*16 + (__builtin_ctzll(mask) >> 2); + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskab), 1)) + return i + 1*16 + (__builtin_ctzll(mask) >> 2); + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskcd), 0)) + return i + 2*16 + (__builtin_ctzll(mask) >> 2); + if(const std::uint64_t mask = vgetq_lane_u64(vreinterpretq_u64_u8(maskcd), 1)) + return i + 3*16 + (__builtin_ctzll(mask) >> 2); + CORRADE_INTERNAL_DEBUG_ASSERT_UNREACHABLE(); /* LCOV_EXCL_LINE */ + } + } + + /* Handle remaining less than four vectors */ + for(; i + 16 <= end; i += 16) + if(const char* const found = findCharacterSingleVectorUnaligned(Cpu::Neon, i, vn1)) + return found; + + /* Handle remaining less than a vector with an unaligned search, again + overlapping back with the previous already-searched elements */ + if(i < end) { + CORRADE_INTERNAL_DEBUG_ASSERT(i + 16 > end); + i = end - 16; + return findCharacterSingleVectorUnaligned(Cpu::Neon, i, vn1); + } + + return static_cast(nullptr); + }; +} +#endif + #ifdef CORRADE_ENABLE_SIMD128 /* WASM doesn't differentiate between aligned and unaligned load, it's always unaligned :( */ diff --git a/src/Corrade/Containers/Test/StringViewBenchmark.cpp b/src/Corrade/Containers/Test/StringViewBenchmark.cpp index f55acab51..0ce2a15a4 100644 --- a/src/Corrade/Containers/Test/StringViewBenchmark.cpp +++ b/src/Corrade/Containers/Test/StringViewBenchmark.cpp @@ -100,6 +100,9 @@ const struct { #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) {Cpu::Avx2|Cpu::Bmi1}, #endif + #ifdef CORRADE_ENABLE_NEON + {Cpu::Neon}, + #endif #ifdef CORRADE_ENABLE_SIMD128 {Cpu::Simd128}, #endif @@ -117,6 +120,9 @@ const struct { {Cpu::Avx2|Cpu::Bmi1, 15}, {Cpu::Avx2|Cpu::Bmi1, 31}, #endif + #ifdef CORRADE_ENABLE_NEON + {Cpu::Neon, 15}, + #endif #ifdef CORRADE_ENABLE_SIMD128 {Cpu::Simd128, 15}, #endif diff --git a/src/Corrade/Containers/Test/StringViewTest.cpp b/src/Corrade/Containers/Test/StringViewTest.cpp index c4f38033f..b0ee0345c 100644 --- a/src/Corrade/Containers/Test/StringViewTest.cpp +++ b/src/Corrade/Containers/Test/StringViewTest.cpp @@ -225,6 +225,9 @@ const struct { #if defined(CORRADE_ENABLE_AVX2) && defined(CORRADE_ENABLE_BMI1) {Cpu::Avx2|Cpu::Bmi1, 32}, #endif + #ifdef CORRADE_ENABLE_NEON + {Cpu::Neon, 16}, + #endif #ifdef CORRADE_ENABLE_SIMD128 {Cpu::Simd128, 16}, #endif From 86bb47ffa9c3f8dc39761615acc6845f68e7198b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 1 Aug 2022 17:35:33 +0200 Subject: [PATCH 79/79] Containers: more perf TODOs. --- src/Corrade/Containers/StringView.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Corrade/Containers/StringView.cpp b/src/Corrade/Containers/StringView.cpp index 2957d3c93..64db6af34 100644 --- a/src/Corrade/Containers/StringView.cpp +++ b/src/Corrade/Containers/StringView.cpp @@ -363,7 +363,8 @@ CORRADE_UTILITY_CPU_MAYBE_UNUSED CORRADE_ENABLE(AVX2,BMI1) typename std::decay(i) + 0*16); const uint8x16_t b = vld1q_u8(reinterpret_cast(i) + 1*16); const uint8x16_t c = vld1q_u8(reinterpret_cast(i) + 2*16);