diff --git a/.clang-format b/.clang-format index 70b1fad4..f000e4ac 100644 --- a/.clang-format +++ b/.clang-format @@ -34,3 +34,15 @@ CompactNamespaces: true BinPackArguments: false BinPackParameters: false +# CUDA-specific additions +AttributeMacros: + - "__global__" + - "__device__" + - "__host__" + - "__constant__" + - "__shared__" + - "__managed__" + - "__restrict__" + - "__launch_bounds__" + - "__forceinline__" + - "__noinline__" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..925646d3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +/.github/workflows/* @azimafroozeh diff --git a/.github/workflows/galp.yaml b/.github/workflows/galp.yaml new file mode 100644 index 00000000..1ee5a36c --- /dev/null +++ b/.github/workflows/galp.yaml @@ -0,0 +1,114 @@ +# .github/workflows/galp.yml + +name: GALP + +run-name: >- + ${{ github.workflow }} on ${{ github.ref_name }} (#${{ github.run_number }}) + — ${{ github.event.pull_request.title || github.event.head_commit.message }} + +on: + push: + +permissions: + contents: read + +jobs: + president-build-test: + if: ${{ github.actor == 'azimafroozeh' }} + name: CMake CI on president + runs-on: + - self-hosted + - Linux + - X64 + - president + + steps: + # ──────────────────────────────────────────────── + # 0. Checkout + # ──────────────────────────────────────────────── + - name: Checkout source + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # ──────────────────────────────────────────────── + # 0a. Generate synthetic dataset + # ──────────────────────────────────────────────── + - name: Generate synthetic dataset + uses: ./.github/actions/generate-dataset + + # ──────────────────────────────────────────────── + # 1. Quick sanity of the runner + # ──────────────────────────────────────────────── + - name: 🔎 Env dump + run: | + set -x + hostname + echo "Runner label: president" + echo "CUDA path: $CUDA_LIBRARY_PATH" + echo "NVCOMP_INC: $NVCOMP_INC" + echo "NVCOMP_LIB: $NVCOMP_LIB" + which nvidia-smi || true + which nvcc || true + which clang++ || true + which g++ || true + which cmake || true + which make || true + + # ──────────────────────────────────────────────── + # 2. Verify required tools + # ──────────────────────────────────────────────── + - name: ✅ Verify tools + run: | + for cmd in nvidia-smi nvcc clang++ g++ cmake make; do + command -v "$cmd" >/dev/null || { echo "::error::$cmd missing"; exit 127; } + done + + - name: Version outputs + run: | + nvidia-smi + nvcc --version + clang++ --version + g++ --version + cmake --version + make --version + + # ──────────────────────────────────────────────── + # 3. Configure with CMake + # ──────────────────────────────────────────────── + - name: 🔄 CMake configure + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DFLS_ENABLE_GALP_TESTING_AND_BENCHMARKING=ON + + # ──────────────────────────────────────────────── + # 4. Build (all targets) + # ──────────────────────────────────────────────── + - name: 🏗️ Build + run: cmake --build build --parallel + + # ──────────────────────────────────────────────── + # 5. Run unit tests + # ──────────────────────────────────────────────── + - name: 🧪 Run tests + run: | + cd build + ctest --output-on-failure + + # ──────────────────────────────────────────────── + # 6. Run benchmarks (no label filter) + # ──────────────────────────────────────────────── + - name: ⚡ Run benchmarks + run: | + cd build + ctest --output-on-failure || true + + # ──────────────────────────────────────────────── + # 7. Smoke‑test helper binaries + # ──────────────────────────────────────────────── + - name: 🚀 Smoke‑test binaries + run: | + ./build/galp/benchmark/micro-benchmarks --help || true + ./build/galp/benchmark/compressors-benchmarks --help || true + shell: bash diff --git a/CMakeLists.txt b/CMakeLists.txt index b040fdca..b3b01e81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,7 +108,7 @@ endif () option(FLS_BUILD_TESTING "Build Test" OFF) option(FLS_BUILD_BENCHMARKING "Enable Benchmark Build" OFF) option(FLS_BUILD_EXAMPLES "Build Examples" OFF) -option(FLS_BUILD_GPU "Build GPU" OFF) +option(FLS_BUILD_CUDA "Build FastLanes CUDA reader" OFF) option(FLS_BUILD_GTEST "Build GTEST" OFF) option(FLS_BUILD_PYTHON "Build FastLanes Python bindings" OFF) option(FLS_BUILD_SHARED_LIBS "Build FastLanes library as shared" OFF) @@ -120,13 +120,13 @@ option(FLS_ENABLE_TABLE_LOG "Enable Table Log" OFF) option(FLS_ENABLE_VERBOSE_OUTPUT "Enable verbose output" OFF) option(FLS_ENABLE_DATA "Enable Data" OFF) option(FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING "Build FSST" OFF) +option(FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING "Build GALP" OFF) # Definitions: --------------------------------------------------------------------------------------------------------- add_compile_definitions(FLS_CMAKE_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}") - # GTEST : -------------------------------------------------------------------------------------------------------------- -if (FLS_BUILD_TESTING OR FLS_BUILD_GPU OR FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING) +if (FLS_BUILD_TESTING OR FLS_BUILD_CUDA OR FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING OR FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING) message("---------------------------------------------------------------------------------------------------------") message("-- FLS: Building GTEST:") include(GoogleTest) @@ -182,8 +182,21 @@ if (true) endif () +# CUDA : --------------------------------------------------------------------------------------------------------------- +if (FLS_BUILD_CUDA) + message("---------------------------------------------------------------------------------------------------------") + message("--: Building CU-FastLanes Lib.") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/cuda) +endif () + + # DATA : --------------------------------------------------------------------------------------------------------------- -if (FLS_BUILD_TESTING OR FLS_BUILD_BENCHMARKING OR FLS_BUILD_EXAMPLES OR FLS_ENABLE_DATA) +if (FLS_BUILD_TESTING OR + FLS_BUILD_BENCHMARKING OR + FLS_BUILD_EXAMPLES OR + FLS_ENABLE_DATA OR + FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING OR + FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING) message("---------------------------------------------------------------------------------------------------------") message("-- FLS: Enabling Data.") @@ -276,7 +289,6 @@ if (FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING) add_subdirectory(fsst) endif () - # Example : ------------------------------------------------------------------------------------------------------------ if (FLS_BUILD_EXAMPLES) message("---------------------------------------------------------------------------------------------------------") @@ -284,13 +296,6 @@ if (FLS_BUILD_EXAMPLES) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples) endif () -# GPU : ---------------------------------------------------------------------------------------------------------------- -if (FLS_BUILD_GPU) - message("---------------------------------------------------------------------------------------------------------") - message("- GPU:") - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/gpu) -endif () - # Playground : --------------------------------------------------------------------------------------------------------- if (EXISTS "${CMAKE_SOURCE_DIR}/playground/CMakeLists.txt") message("---------------------------------------------------------------------------------------------------------") @@ -305,4 +310,9 @@ if (FLS_BUILD_PYTHON) add_subdirectory(python) # your module’s CMakeLists.txt endif () - +# FSST: ---------------------------------------------------------------------------------------------------------------- +if (FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING) + message("---------------------------------------------------------------------------------------------------------") + message("-- FLS: Adding GALP Testing and Benchmarking.") + add_subdirectory(galp) +endif () \ No newline at end of file diff --git a/README.md b/README.md index 3f99d030..318e43fd 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ FastLanes features: -- **Fully SIMD-friendly with zero explicit SIMD instructions**: -- **Zero dependencies**: - - no external libraries required +* **Fully SIMD-friendly with zero explicit SIMD instructions** +* **Zero dependencies**: + + * no external libraries required --- @@ -97,7 +98,7 @@ fn main() -> Result<()> { ## Documentation -- [Typing System](docs/typing_system.md) +* [Typing System](docs/typing_system.md) --- @@ -121,6 +122,12 @@ fn main() -> Result<()> { * [Read the paper](https://doi.org/10.1145/3626717) * [Source code](https://github.com/cwida/ALP) +* **Sven Hielke Hepkema, Azim Afroozeh, Charlotte Felius, Peter Boncz & Stefan Manegold**, “G‑ALP: Rethinking Light‑weight Encodings for GPUs,” *DaMoN ’25*, July 2025 + + * [Read the paper](https://dl.acm.org/doi/pdf/10.1145/3736227.3736242) + * [Source code](https://github.com/cwida/FastLanesGpu-Damon2025) + + --- ## How to Cite @@ -161,6 +168,17 @@ If you use FastLanes in your research or projects, please cite: organization = {ACM}, doi = {10.1145/3626717} } + +@inproceedings{hepke2025galp, + author = {Hepkema, Sven Hielke and Afroozeh, Azim and Felius, Charlotte and Boncz, Peter and Manegold, Stefan}, + title = {G‑ALP: Rethinking Light‑weight Encodings for GPUs}, + booktitle = {DaMoN ’25: Proceedings of the 21st International Workshop on Data Management on New Hardware}, + pages = {11:1--11:10}, + month = jul, + year = {2025}, + organization = {ACM}, + url = {https://dl.acm.org/doi/pdf/10.1145/3736227.3736242} +} ``` --- diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index e2b4270a..6909005b 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -30,3 +30,5 @@ target_link_libraries(bench_share_schema PUBLIC FastLanes::benchmarker) add_executable(bench_compression_time bench_compression_time.cpp) target_link_libraries(bench_compression_time PUBLIC FastLanes::benchmarker) + +add_subdirectory(bench_galp) \ No newline at end of file diff --git a/benchmark/bench_galp/CMakeLists.txt b/benchmark/bench_galp/CMakeLists.txt new file mode 100644 index 00000000..36601ef4 --- /dev/null +++ b/benchmark/bench_galp/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(bench_galp bench_data_parallelize_patch.cpp) +target_link_libraries(bench_galp PUBLIC FastLanes::benchmarker) \ No newline at end of file diff --git a/benchmark/bench_galp/bench_data_parallelize_patch.cpp b/benchmark/bench_galp/bench_data_parallelize_patch.cpp new file mode 100644 index 00000000..87a255fb --- /dev/null +++ b/benchmark/bench_galp/bench_data_parallelize_patch.cpp @@ -0,0 +1,230 @@ +// ───────────────────────────────────────────────────────────────────────────── +// patch_bench.cpp – unified benchmark driver +// ───────────────────────────────────────────────────────────────────────────── +#include "cycle_counter.hpp" +#include "fastlanes.hpp" +#include "fls/cuda/common.hpp" +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Random helpers +// --------------------------------------------------------------------------- +namespace fastlanes::helper { + +inline std::mt19937_64& rng() { + static std::mt19937_64 eng {std::random_device {}()}; + return eng; +} + +template +std::enable_if_t> fill_random_exceptions(PT* dst, std::size_t n) { + std::uniform_int_distribution dist(std::numeric_limits::min(), std::numeric_limits::max()); + for (std::size_t i = 0; i < n; ++i) + dst[i] = dist(rng()); +} + +template +std::enable_if_t> fill_random_exceptions(PT* dst, std::size_t n) { + std::uniform_real_distribution dist(PT(-1), PT(1)); + for (std::size_t i = 0; i < n; ++i) + dst[i] = dist(rng()); +} + +inline void fill_random_positions(uint16_t* dst, std::size_t n) { + std::uniform_int_distribution dist(0, 1023); + for (std::size_t i = 0; i < n; ++i) + dst[i] = dist(rng()); +} + +} // namespace fastlanes::helper + +// --------------------------------------------------------------------------- +// Kernels +// --------------------------------------------------------------------------- +namespace fastlanes { +template +static inline void new_data_parallelizer(const PT* in_exc_arr, + const uint16_t* in_pos_arr, + n_t n_exceptions, + PT* out_exc_arr, + uint16_t* out_offset, + uint16_t* out_pos_arr, + uint8_t* out_count) { + constexpr n_t N_LANES = cuda::get_n_lanes(); + constexpr n_t MASK = N_LANES - 1; + static_assert((N_LANES & MASK) == 0, "N_LANES must be power-of-two"); + + for (n_t l = 0; l < N_LANES; ++l) + out_count[l] = 0; + for (n_t i = 0; i < n_exceptions; ++i) + ++out_count[in_pos_arr[i] & MASK]; + + uint16_t running = 0; + for (n_t l = 0; l < N_LANES; ++l) { + out_offset[l] = running; + running += out_count[l]; + } + + for (n_t i = 0; i < n_exceptions; ++i) { + const n_t lane = in_pos_arr[i] & MASK; + const uint16_t dst = out_offset[lane]++; + out_exc_arr[dst] = in_exc_arr[i]; + out_pos_arr[dst] = in_pos_arr[i]; + } +} + +template +static inline void old_data_parallelizer(const PT* in_exc_arr, + const uint16_t* in_pos_arr, + n_t n_exceptions, + PT* out_exc_arr, + uint16_t* out_offset, + uint16_t* out_pos_arr, + uint8_t* out_count) { + constexpr n_t N_LANES = cuda::get_n_lanes(); + constexpr n_t VALUES_PER_LANE = cuda::get_values_per_lane(); + + PT vec_exc[CFG::VEC_SZ]; + PT vec_pos_proxy[CFG::VEC_SZ]; + uint16_t n_local_exp[N_LANES] {}; + + for (n_t exc_idx = 0; exc_idx < n_exceptions; ++exc_idx) { + PT exc = in_exc_arr[exc_idx]; + uint16_t position = in_pos_arr[exc_idx]; + n_t lane = position % N_LANES; + auto& count = n_local_exp[lane]; + vec_exc[lane * VALUES_PER_LANE + count] = exc; + vec_pos_proxy[lane * VALUES_PER_LANE + count] = position; + ++count; + } + + n_t global_idx = 0; + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_offset[lane] = static_cast(global_idx); + for (n_t k = 0; k < n_local_exp[lane]; ++k) { + out_exc_arr[global_idx] = vec_exc[lane * VALUES_PER_LANE + k]; + out_pos_arr[global_idx] = static_cast(vec_pos_proxy[lane * VALUES_PER_LANE + k]); + ++global_idx; + } + out_count[lane] = static_cast(n_local_exp[lane]); + } +} + +// Thin adapter for Patch kernel +template +static inline void try_this_data_parallelizer(const PT* in_exc, + const uint16_t* in_pos, + n_t n_exc, + PT* out_exc, + uint16_t* out_off, + uint16_t* out_pos, + uint8_t* out_cnt) { + constexpr n_t N_LANES = cuda::get_n_lanes(); + constexpr n_t MASK = N_LANES - 1; + static_assert((N_LANES & MASK) == 0); + + for (n_t lane = 0; lane < N_LANES; ++lane) + out_cnt[lane] = 0; + + for (n_t i = 0; i < n_exc; ++i) + ++out_cnt[in_pos[i] & MASK]; + + uint16_t prefix = 0; + PT* exc_ptr[N_LANES]; + uint16_t* pos_ptr[N_LANES]; +#pragma unroll + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_off[lane] = prefix; + exc_ptr[lane] = out_exc + prefix; + pos_ptr[lane] = out_pos + prefix; + prefix += out_cnt[lane]; + } + + for (n_t i = 0; i < n_exc; ++i) { + n_t lane = in_pos[i] & MASK; + *exc_ptr[lane]++ = in_exc[i]; + *pos_ptr[lane]++ = in_pos[i]; + } +} + +// --------------------------------------------------------------------------- +// Benchmark plumbing +// --------------------------------------------------------------------------- +template +static benchmark::BenchmarkReporter::Run run_patch_parallelize(int id, std::string name, n_t N_EXC) { + constexpr std::size_t ALIGN = 64; +#ifdef NDEBUG + constexpr std::uint64_t ITER = 30'000'000; +#else + constexpr std::uint64_t ITER = 1; +#endif + + auto in_exc = static_cast(::operator new[](N_EXC * sizeof(PT), std::align_val_t {ALIGN})); + auto out_exc = static_cast(::operator new[](N_EXC * sizeof(PT), std::align_val_t {ALIGN})); + auto in_pos = static_cast(::operator new[](N_EXC * sizeof(uint16_t), std::align_val_t {ALIGN})); + auto out_pos = static_cast(::operator new[](N_EXC * sizeof(uint16_t), std::align_val_t {ALIGN})); + + constexpr n_t N_LANES = cuda::get_n_lanes(); + auto out_off = static_cast(::operator new[](N_LANES * sizeof(uint16_t), std::align_val_t {ALIGN})); + auto out_cnt = static_cast(::operator new[](N_LANES * sizeof(uint8_t), std::align_val_t {ALIGN})); + + helper::fill_random_exceptions(in_exc, N_EXC); + helper::fill_random_positions(in_pos, N_EXC); + + const auto start = benchmark::cycleclock::Now(); + for (std::uint64_t i = 0; i < ITER; ++i) + Fn(in_exc, in_pos, N_EXC, out_exc, out_off, out_pos, out_cnt); + const auto cycles = benchmark::cycleclock::Now() - start; + + ::operator delete[](in_exc, std::align_val_t {ALIGN}); + ::operator delete[](out_exc, std::align_val_t {ALIGN}); + ::operator delete[](in_pos, std::align_val_t {ALIGN}); + ::operator delete[](out_pos, std::align_val_t {ALIGN}); + ::operator delete[](out_off, std::align_val_t {ALIGN}); + ::operator delete[](out_cnt, std::align_val_t {ALIGN}); + + return benchmark::BenchmarkReporter::Run {id, name, ITER, double(cycles) / double(ITER)}; +} + +// --------------------------------------------------------------------------- +// Register & run +// --------------------------------------------------------------------------- +static void benchmark_all(benchmark::Benchmark& bench) { + int idx = 0; + for (n_t exc = 0; exc <= 20; ++exc) { + // float kernels + bench.Run( + run_patch_parallelize>(idx++, "new_float" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "old_float" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "try" + std::to_string(exc), exc)); + + // double kernels + bench.Run( + run_patch_parallelize>(idx++, "new_dbl" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "old_dbl" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "try" + std::to_string(exc), exc)); + } +} +} // namespace fastlanes +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- +int main() { + benchmark::Benchmark bench = benchmark::create("patch") + .save() + .at(std::string(FLS_CMAKE_SOURCE_DIR) + "/benchmark/result/galp") + .print() + .add_extra_info(benchmark::CmakeInfo::getCmakeInfo()); + + fastlanes::benchmark_all(bench); + return 0; +} diff --git a/benchmark/result/galp/benchmark_new_vs_old.png b/benchmark/result/galp/benchmark_new_vs_old.png new file mode 100644 index 00000000..e841fad4 Binary files /dev/null and b/benchmark/result/galp/benchmark_new_vs_old.png differ diff --git a/benchmark/result/galp/patch.csv b/benchmark/result/galp/patch.csv new file mode 100644 index 00000000..e57c9bf6 --- /dev/null +++ b/benchmark/result/galp/patch.csv @@ -0,0 +1,127 @@ +benchmark_number,name,iterations,cycles_per_tuple +0,new_float0,30000000,36.9986 +1,old_float0,30000000,54.7735 +2,try0,30000000,74.7888 +3,new_dbl0,30000000,17.7192 +4,old_dbl0,30000000,50.1561 +5,try0,30000000,39.5836 +6,new_float1,30000000,42.4431 +7,old_float1,30000000,70.051 +8,try1,30000000,80.1325 +9,new_dbl1,30000000,25.5231 +10,old_dbl1,30000000,41.4336 +11,try1,30000000,44.8089 +12,new_float2,30000000,46.0671 +13,old_float2,30000000,95.3122 +14,try2,30000000,84.1434 +15,new_dbl2,30000000,27.8461 +16,old_dbl2,30000000,53.6348 +17,try2,30000000,48.8947 +18,new_float3,30000000,49.279 +19,old_float3,30000000,103.336 +20,try3,30000000,90.7655 +21,new_dbl3,30000000,30.8936 +22,old_dbl3,30000000,67.452 +23,try3,30000000,52.9569 +24,new_float4,30000000,52.5484 +25,old_float4,30000000,118.592 +26,try4,30000000,91.8901 +27,new_dbl4,30000000,35.3823 +28,old_dbl4,30000000,68.8603 +29,try4,30000000,58.1011 +30,new_float5,30000000,56.4755 +31,old_float5,30000000,120.493 +32,try5,30000000,97.1593 +33,new_dbl5,30000000,37.6887 +34,old_dbl5,30000000,100.8 +35,try5,30000000,62.4423 +36,new_float6,30000000,58.9193 +37,old_float6,30000000,133.728 +38,try6,30000000,101.672 +39,new_dbl6,30000000,41.8799 +40,old_dbl6,30000000,106.015 +41,try6,30000000,66.32 +42,new_float7,30000000,61.5631 +43,old_float7,30000000,156.774 +44,try7,30000000,106.072 +45,new_dbl7,30000000,45.1069 +46,old_dbl7,30000000,110.342 +47,try7,30000000,71.5572 +48,new_float8,30000000,66.8794 +49,old_float8,30000000,159.354 +50,try8,30000000,108.794 +51,new_dbl8,30000000,47.6713 +52,old_dbl8,30000000,112.159 +53,try8,30000000,75.1476 +54,new_float9,30000000,67.7994 +55,old_float9,30000000,174.6 +56,try9,30000000,114.291 +57,new_dbl9,30000000,52.0896 +58,old_dbl9,30000000,123.841 +59,try9,30000000,79.5354 +60,new_float10,30000000,71.2754 +61,old_float10,30000000,186.312 +62,try10,30000000,118.533 +63,new_dbl10,30000000,57.6601 +64,old_dbl10,30000000,136.717 +65,try10,30000000,85.798 +66,new_float11,30000000,81.8039 +67,old_float11,30000000,193.181 +68,try11,30000000,123.711 +69,new_dbl11,30000000,60.9535 +70,old_dbl11,30000000,150.633 +71,try11,30000000,89.8322 +72,new_float12,30000000,77.1956 +73,old_float12,30000000,195.674 +74,try12,30000000,129.432 +75,new_dbl12,30000000,62.5729 +76,old_dbl12,30000000,135.393 +77,try12,30000000,94.2064 +78,new_float13,30000000,80.3247 +79,old_float13,30000000,207.773 +80,try13,30000000,130.284 +81,new_dbl13,30000000,70.5584 +82,old_dbl13,30000000,156.095 +83,try13,30000000,99.3467 +84,new_float14,30000000,84.7039 +85,old_float14,30000000,230.406 +86,try14,30000000,136.923 +87,new_dbl14,30000000,73.6667 +88,old_dbl14,30000000,192.454 +89,try14,30000000,105.223 +90,new_float15,30000000,89.7811 +91,old_float15,30000000,236.895 +92,try15,30000000,138.972 +93,new_dbl15,30000000,77.1329 +94,old_dbl15,30000000,187.898 +95,try15,30000000,107.881 +96,new_float16,30000000,90.1445 +97,old_float16,30000000,236.516 +98,try16,30000000,145.209 +99,new_dbl16,30000000,77.5095 +100,old_dbl16,30000000,186.772 +101,try16,30000000,112.768 +102,new_float17,30000000,96.1895 +103,old_float17,30000000,236.044 +104,try17,30000000,149.086 +105,new_dbl17,30000000,93.9837 +106,old_dbl17,30000000,174.124 +107,try17,30000000,116.276 +108,new_float18,30000000,98.1571 +109,old_float18,30000000,260.646 +110,try18,30000000,152.978 +111,new_dbl18,30000000,91.5388 +112,old_dbl18,30000000,174.01 +113,try18,30000000,125.208 +114,new_float19,30000000,111.783 +115,old_float19,30000000,276.021 +116,try19,30000000,161.24 +117,new_dbl19,30000000,133.839 +118,old_dbl19,30000000,188.064 +119,try19,30000000,136.332 +120,new_float20,30000000,104.414 +121,old_float20,30000000,266.528 +122,try20,30000000,161.393 +123,new_dbl20,30000000,94.7421 +124,old_dbl20,30000000,199.619 +125,try20,30000000,143.522 diff --git a/benchmark/result/galp/patch.metadata b/benchmark/result/galp/patch.metadata new file mode 100644 index 00000000..9de73a3c --- /dev/null +++ b/benchmark/result/galp/patch.metadata @@ -0,0 +1,26 @@ +Run on (10 X 4500 MHz CPU s) +CPU Caches: + L1 Data 32 KiB (x10) + L1 Instruction 32 KiB (x10) + L2 Unified 1024 KiB (x10) + L3 Unified 14080 KiB (x1) +***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. +cmake info: + source_dir: SOURCE_DIR + cmake_osx_architecturesCMAKE_OSX_ARCHITECTURES + cmake_host_system_processor: CMAKE_HOST_SYSTEM_PROCESSOR + cmake_system_processor: CMAKE_SYSTEM_PROCESSOR + cmake_host_system_name: CMAKE_HOST_SYSTEM_NAME + cmake_system_name: CMAKE_SYSTEM_NAME + cmake_c_compiler: CMAKE_C_COMPILER + cmake_cxx_compiler: CMAKE_CXX_COMPILER + cmake_cxx_compiler_id: CMAKE_CXX_COMPILER_ID + cmake_cxx_compiler_version: CMAKE_CXX_COMPILER_VERSION + cmake_crosscompiling: CMAKE_CROSSCOMPILING + cmake_cxx_flags_debug: CMAKE_CXX_FLAGS_DEBUG + cmake_cxx_flags_release: CMAKE_CXX_FLAGS_RELEASE + cmake_build_type: CMAKE_BUILD_TYPE + cmake_toolchain_file: CMAKE_TOOLCHAI +target info: + target_name: TARGET_NAME + target_compile_options: TARGET_COMPILE_OPTIONS diff --git a/src/cor/exp_pool.cpp b/benchmark/result/galp/patch/README.md similarity index 100% rename from src/cor/exp_pool.cpp rename to benchmark/result/galp/patch/README.md diff --git a/benchmark/result/galp/plot.py b/benchmark/result/galp/plot.py new file mode 100644 index 00000000..95dc0df2 --- /dev/null +++ b/benchmark/result/galp/plot.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Benchmark visualisation helper (combined float & dbl subplots, new vs old). + +* Accepts CSVs (with or without header) in new or legacy format. +* Extracts exception counts from trailing digits in names. +* Classifies each entry as float vs dbl based on name suffix. +* Plots both float and dbl benchmarks side-by-side in a single figure. +* Only compares "new" vs "old" variants, dropping "try" entries. +* Prints average cycles per run and speedup factor of new over old. +* Saves the combined figure as a PNG file. + +Usage:: + python plot.py path/to/patch.csv + # Plot will be saved as benchmark_new_vs_old.png +""" + +from __future__ import annotations +import sys +from pathlib import Path + +import matplotlib.pyplot as plt +import pandas as pd + + +def _canonicalise_columns(df: pd.DataFrame) -> pd.DataFrame: + col_map = {"benchmark_number": "id", "cycles_per_tuple": "cycles"} + df = df.rename(columns=col_map, errors="ignore") + required = {"id", "name", "iterations", "cycles"} + missing = required.difference(df.columns) + if missing: + raise ValueError(f"Missing required columns: {', '.join(sorted(missing))}") + return df + + +def load_and_prepare(path: str | Path = "patch.csv") -> pd.DataFrame: + """Load CSV and return DataFrame with exceptions, cycles, version, and type.""" + df = pd.read_csv(path) + if "benchmark_number" not in df.columns: + df = pd.read_csv( + path, + header=None, + names=["benchmark_number", "name", "iterations", "cycles_per_tuple"], + ) + df = _canonicalise_columns(df) + + # Ensure id is integer + df["id"] = pd.to_numeric(df["id"], errors="coerce").astype(pd.Int64Dtype()) + df = df.dropna(subset=["id"]) + df["id"] = df["id"].astype(int) + + # Extract exception count (trailing digits) + df["exceptions"] = ( + df["name"].str.extract(r"(\d+)$")[0] + .astype(pd.Int64Dtype()) + ) + df = df.dropna(subset=["exceptions"]) + df["exceptions"] = df["exceptions"].astype(int) + + # Extract version prefix (new/old), drop 'try' + df["version"] = df["name"].str.extract(r"^(new|old)")[0] + df = df.dropna(subset=["version"]) + + # Classify type by name suffix: 'float' or 'dbl' + df["type"] = df["name"].apply( + lambda s: "float" if "float" in s else ("dbl" if "dbl" in s else None) + ) + df = df.dropna(subset=["type"]) + + # Convert cycles to numeric + df["cycles"] = pd.to_numeric(df["cycles"], errors="coerce") + + return df + + +def plot_benchmarks(df: pd.DataFrame, save_path: Path = Path("benchmark_new_vs_old.png")) -> None: + """Plot float and dbl benchmarks in a single figure with two subplots and save.""" + color_map = {"new": "green", "old": "red"} + types = ["float", "dbl"] + + fig, axes = plt.subplots(1, 2, figsize=(16, 5), sharey=True) + for ax, typ in zip(axes, types): + sub = df[df["type"] == typ] + if sub.empty: + ax.set_visible(False) + continue + pivot = ( + sub.pivot_table( + index="exceptions", columns="version", values="cycles", aggfunc="mean" + ) + .sort_index() + ) + for version in ["new", "old"]: + if version in pivot.columns: + series = pivot[version] + ax.plot( + series.index, + series.values, + marker="o", + label=version, + color=color_map.get(version), + ) + ax.set_xticks(pivot.index) + ax.set_xlabel("Number of Exceptions") + ax.set_title(f"{typ.capitalize()} benchmarks: new vs old") + ax.grid(True, linestyle="--", alpha=0.5) + ax.legend(title="Version") + + axes[0].set_ylabel("Cycles per Run") + fig.suptitle("Benchmark: new vs old (float & dbl)") + plt.tight_layout(rect=[0, 0, 1, 0.95]) + # Save figure + fig.savefig(save_path) + print(f"Plot saved to {save_path}") + plt.show() + + +def main(argv: list[str] | None = None) -> None: + import argparse + parser = argparse.ArgumentParser(description="Plot float and dbl benchmark results.") + parser.add_argument("csv", nargs="?", default="patch.csv", help="CSV file to read") + parser.add_argument("--out", help="Output PNG path", default="benchmark_new_vs_old.png") + ns = parser.parse_args(argv) + path = Path(ns.csv) + if not path.exists(): + parser.error(f"CSV not found: {path}") + df = load_and_prepare(path) + if df.empty: + parser.error("No valid data to plot.") + + # Compute and print average cycles and speedup factor of new over old + for typ in ["float", "dbl"]: + sub = df[df["type"] == typ] + if sub.empty: + continue + pivot = ( + sub.pivot_table( + index="exceptions", columns="version", values="cycles", aggfunc="mean" + ).sort_index() + ) + if "new" in pivot.columns and "old" in pivot.columns: + old = pivot["old"] + new = pivot["new"] + avg_new = new.mean() + avg_old = old.mean() + times = old / new + avg_times = times.mean() + print(f"{typ.capitalize()}: average cycles per run — new: {avg_new:.1f}, old: {avg_old:.1f}.") + print(f"{typ.capitalize()}: new is on average {avg_times:.2f}× faster than old.") + + plot_benchmarks(df, Path(ns.out)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt new file mode 100644 index 00000000..328e7879 --- /dev/null +++ b/cuda/CMakeLists.txt @@ -0,0 +1,44 @@ +cmake_minimum_required(VERSION 3.18 FATAL_ERROR) +project(cuda_info LANGUAGES CXX CUDA) + +# --- CUDA compiler info via CMake --- +message(STATUS "CMAKE_CUDA_COMPILER: ${CMAKE_CUDA_COMPILER}") +message(STATUS "CMAKE_CUDA_COMPILER_ID: ${CMAKE_CUDA_COMPILER_ID}") +message(STATUS "CMAKE_CUDA_COMPILER_VERSION: ${CMAKE_CUDA_COMPILER_VERSION}") +message(STATUS "CMAKE_CUDA_STANDARD: ${CMAKE_CUDA_STANDARD}") +message(STATUS "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}") + +# --- Find the CUDA toolkit (modern CMake) --- +find_package(CUDAToolkit REQUIRED) +message(STATUS "CUDAToolkit version: ${CUDAToolkit_VERSION}") +message(STATUS "CUDAToolkit include dirs: ${CUDAToolkit_INCLUDE_DIRS}") +message(STATUS "CUDAToolkit libraries: ${CUDAToolkit_LIBRARIES}") + +# --- Locate nvcc and dump version --- +find_program(NVCC_EXECUTABLE nvcc REQUIRED) +message(STATUS "nvcc executable: ${NVCC_EXECUTABLE}") +execute_process( + COMMAND "${NVCC_EXECUTABLE}" --version + OUTPUT_VARIABLE _nvcc_version + OUTPUT_STRIP_TRAILING_WHITESPACE +) +message(STATUS "nvcc --version output:\n${_nvcc_version}") + +# --- Echo common CUDA-related env vars --- +foreach (_env_var CUDA_PATH CUDA_HOME CUDA_TOOLKIT_ROOT_DIR) + if (DEFINED ENV{${_env_var}}) + message(STATUS "${_env_var}: $ENV{${_env_var}}") + endif () +endforeach () + +# --- Point at your subdirectory source file --- +add_executable(cuda_info_dummy + ${CMAKE_CURRENT_SOURCE_DIR}/src/cuda_reader.cu +) + +# --- Link against CUDA runtime --- +target_link_libraries(cuda_info_dummy PRIVATE CUDA::cudart) +set_target_properties(cuda_info_dummy PROPERTIES + CUDA_STANDARD 11 + CUDA_STANDARD_REQUIRED YES +) diff --git a/cuda/src/cuda_reader.cu b/cuda/src/cuda_reader.cu new file mode 100644 index 00000000..d60cb546 --- /dev/null +++ b/cuda/src/cuda_reader.cu @@ -0,0 +1,19 @@ +#include + +// A simple CUDA Hello World example +// This kernel prints a greeting from each thread on the GPU +__global__ void hello_from_gpu() { + int tid = threadIdx.x; + printf("Hello World from GPU! Thread %d says hi.\n", tid); +} + +int main() { + // Launch the kernel with 4 threads in one block + hello_from_gpu<<<1, 4>>>(); + + // Wait for GPU to finish before accessing on host + cudaDeviceSynchronize(); + + // And return + return 0; +} diff --git a/data/include/data/fastlanes_data.hpp b/data/include/data/fastlanes_data.hpp index 6f3eb4b8..ce76073a 100644 --- a/data/include/data/fastlanes_data.hpp +++ b/data/include/data/fastlanes_data.hpp @@ -7,6 +7,7 @@ #include "data/example.hpp" #include "data/fannie_mae.hpp" #include "data/fc_bench.hpp" +#include "data/galp.hpp" #include "data/generated.hpp" #include "data/issues.hpp" #include "data/public_bi.hpp" diff --git a/data/include/data/galp.hpp b/data/include/data/galp.hpp new file mode 100644 index 00000000..5ed6bed1 --- /dev/null +++ b/data/include/data/galp.hpp @@ -0,0 +1,64 @@ +#ifndef DATA_GALP_HPP +#define DATA_GALP_HPP + +#include +#include + +namespace fastlanes { + +using galp_dataset_t = std::array, 12>; + +class galp { +public: + // FLOAT precision + static constexpr std::string_view float_galp_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/float/galp/one_vector"}; + static constexpr std::string_view float_galp_single_column {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/float/galp/single_column"}; + + static constexpr std::string_view float_constant_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/float/constant/one_vector"}; + static constexpr std::string_view float_constant_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/float/constant/single_column"}; + + static constexpr std::string_view float_positive_inf_one_vector { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/float/positive_inf/one_vector"}; + static constexpr std::string_view float_positive_inf_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/float/positive_inf/single_column"}; + + // DOUBLE precision + static constexpr std::string_view double_galp_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/double/galp/one_vector"}; + static constexpr std::string_view double_galp_single_column {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/double/galp/single_column"}; + + static constexpr std::string_view double_constant_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/double/constant/one_vector"}; + static constexpr std::string_view double_constant_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/double/constant/single_column"}; + + static constexpr std::string_view double_positive_inf_one_vector { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/double/positive_inf/one_vector"}; + static constexpr std::string_view double_positive_inf_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/double/positive_inf/single_column"}; + + static constexpr galp_dataset_t dataset { + {// Float entries + {"float_galp_one_vector", float_galp_one_vector}, + {"float_galp_single_column", float_galp_single_column}, + {"float_constant_one_vector", float_constant_one_vector}, + {"float_constant_single_column", float_constant_single_column}, + {"float_positive_inf_one_vector", float_positive_inf_one_vector}, + {"float_positive_inf_single_column", float_positive_inf_single_column}, + // Double entries + {"double_galp_one_vector", double_galp_one_vector}, + {"double_galp_single_column", double_galp_single_column}, + {"double_constant_one_vector", double_constant_one_vector}, + {"double_constant_single_column", double_constant_single_column}, + {"double_positive_inf_one_vector", double_positive_inf_one_vector}, + {"double_positive_inf_single_column", double_positive_inf_single_column}}}; +}; + +} // namespace fastlanes + +#endif // DATA_GALP_HPP diff --git a/flatbuffers_schemas/operator_token.fbs b/flatbuffers_schemas/operator_token.fbs index f9130374..6ce564e0 100644 --- a/flatbuffers_schemas/operator_token.fbs +++ b/flatbuffers_schemas/operator_token.fbs @@ -207,4 +207,6 @@ enum OperatorToken : ushort { EXP_RLE_U08_SLPATCH_U16 = 152, EXP_DELTA_U08 = 153, EXP_RLE_U08_U16 = 154, + EXP_GALP_FLT = 155, + EXP_GALP_DBL = 156, } diff --git a/galp/CMakeLists.txt b/galp/CMakeLists.txt new file mode 100644 index 00000000..f83fe8e7 --- /dev/null +++ b/galp/CMakeLists.txt @@ -0,0 +1,92 @@ +enable_language(CUDA) + +# ──────────────────────────────────────────────── +# 0. Global settings +# ──────────────────────────────────────────────── +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_CUDA_SEPARABLE_COMPILATION ON) +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr") #todo FIX ME + +# ──────────────────────────────────────────────── +# 1. CUDA Toolkit +# ──────────────────────────────────────────────── +find_package(CUDAToolkit REQUIRED) + +# ──────────────────────────────────────────────── +# 2. nvCOMP (required) +# ──────────────────────────────────────────────── +find_package(nvcomp CONFIG REQUIRED) + +get_target_property(NVCOMP_VERSION nvcomp::nvcomp VERSION) +get_target_property(NVCOMP_INC_DIRS nvcomp::nvcomp INTERFACE_INCLUDE_DIRECTORIES) +get_target_property(NVCOMP_LIBS nvcomp::nvcomp INTERFACE_LINK_LIBRARIES) + +message(STATUS "nvCOMP version ..........: ${NVCOMP_VERSION}") +message(STATUS "nvCOMP include dirs .....: ${NVCOMP_INC_DIRS}") +message(STATUS "nvCOMP libs .............: ${NVCOMP_LIBS}") + +# ──────────────────────────────────────────────── +# 3. Thrust / CCCL – prefer toolkit copy, fall back to CCCL ≥ 2.3 +# ──────────────────────────────────────────────── +find_path(THRUST_INCLUDE_DIR + NAMES thrust/version.h + thrust/device_uvector.h + HINTS ${CUDAToolkit_INCLUDE_DIRS}) + +if (NOT THRUST_INCLUDE_DIR) + message(FATAL_ERROR "Need Thrust ≥ 2.2 (with device_uvector.h) in CUDA Toolkit") +endif () + +add_library(Thrust::Thrust INTERFACE IMPORTED) +target_include_directories(Thrust::Thrust INTERFACE ${THRUST_INCLUDE_DIR}) + +# ──────────────────────────────────────────────── +# 4. User-tunable options & warnings +# ──────────────────────────────────────────────── +option(USE_CLANG_FOR_HOST "Use clang++-14 for all .cpp files" ON) +option(GALP_VERBOSE_BUILD "Print every compile/link command" OFF) +if (GALP_VERBOSE_BUILD) + set(CMAKE_VERBOSE_MAKEFILE ON) +endif () + +set(COMMON_WARNINGS "") +set(CLANG_ONLY_WARNINGS + -Wno-c++98-compat-local-type-template-args + -Wno-c++98-compat-pedantic + -Wno-c++98-compat + -Wno-padded + -Wno-float-equal + -Wno-global-constructors + -Wno-exit-time-destructors + -Wno-float-conversion + -Wno-sign-conversion + -Wno-implicit-int-float-conversion + -Wno-unused-parameter + -Wno-unused-variable +) + +macro(apply_warnings tgt) + target_compile_options(${tgt} PRIVATE + $<$:${COMMON_WARNINGS}> + $<$,$>:${CLANG_ONLY_WARNINGS}> + $<$:${COMMON_WARNINGS}>) +endmacro() + +add_compile_definitions(FLS_GALP_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}") + +# ──────────────────────────────────────────────── +# 5. GALP core static library +# ──────────────────────────────────────────────── +add_subdirectory(src) + +# ──────────────────────────────────────────────── +# 8. tests +# ──────────────────────────────────────────────── +add_subdirectory(test) + +## ──────────────────────────────────────────────── +## 7. Benchmarks +## ──────────────────────────────────────────────── +add_subdirectory(benchmark) + diff --git a/galp/README.md b/galp/README.md new file mode 100644 index 00000000..3644d519 --- /dev/null +++ b/galp/README.md @@ -0,0 +1,3 @@ +# G-ALP + +Heavily modified version of the original G-ALP codebase at https://github.com/cwida/FastLanesGpu-Damon2025. diff --git a/galp/benchmark/CMakeLists.txt b/galp/benchmark/CMakeLists.txt new file mode 100644 index 00000000..cfd77373 --- /dev/null +++ b/galp/benchmark/CMakeLists.txt @@ -0,0 +1,46 @@ +## ──────────────────────────────────────────────── +## Generated-bindings library +## ──────────────────────────────────────────────── +set(GALP_GENERATED_BINDINGS_SOURCES + generated-bindings/alp-double-decompress_column-bindings.cu + generated-bindings/alp-double-query_column-bindings.cu + generated-bindings/alp-double-query_multi_column-bindings.cu + generated-bindings/alp-float-decompress_column-bindings.cu + generated-bindings/alp-float-query_column-bindings.cu + generated-bindings/alp-float-query_multi_column-bindings.cu + generated-bindings/alpextended-double-decompress_column-bindings.cu + generated-bindings/alpextended-double-query_column-bindings.cu + generated-bindings/alpextended-double-query_multi_column-bindings.cu + generated-bindings/alpextended-float-decompress_column-bindings.cu + generated-bindings/alpextended-float-query_column-bindings.cu + generated-bindings/alpextended-float-query_multi_column-bindings.cu + generated-bindings/bp-uint32_t-decompress_column-bindings.cu + generated-bindings/bp-uint32_t-query_column-bindings.cu + generated-bindings/bp-uint64_t-decompress_column-bindings.cu + generated-bindings/bp-uint64_t-query_column-bindings.cu + generated-bindings/ffor-uint32_t-compute_column-bindings.cu + generated-bindings/ffor-uint32_t-decompress_column-bindings.cu + generated-bindings/ffor-uint32_t-query_column-bindings.cu + generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu + generated-bindings/ffor-uint64_t-compute_column-bindings.cu + generated-bindings/ffor-uint64_t-decompress_column-bindings.cu + generated-bindings/ffor-uint64_t-query_column-bindings.cu + generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu +) +add_library(generated-bindings STATIC ${GALP_GENERATED_BINDINGS_SOURCES}) +target_compile_options(generated-bindings PRIVATE $<$:-expt-relaxed-constexpr>) +target_include_directories(generated-bindings PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CUDAToolkit_INCLUDE_DIRS}) +target_link_libraries(generated-bindings PUBLIC CUDA::cudart nvcomp::nvcomp Thrust::Thrust galp_core) +apply_warnings(generated-bindings) +set_target_properties(generated-bindings PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + + +add_executable(compressors-benchmarks compressors-benchmarks.cu nvcomp/benchmark-compressors.cu nvcomp/nvcomp-compressors.cu) +apply_warnings(compressors-benchmarks) +target_link_libraries(compressors-benchmarks PRIVATE galp_core) + +add_executable(micro-benchmarks micro-benchmarks.cu nvcomp/benchmark-compressors.cu nvcomp/nvcomp-compressors.cu) +apply_warnings(micro-benchmarks) +target_link_libraries(micro-benchmarks PRIVATE galp_core generated-bindings) diff --git a/galp/benchmark/compressors-benchmarks.cu b/galp/benchmark/compressors-benchmarks.cu new file mode 100644 index 00000000..f40e7897 --- /dev/null +++ b/galp/benchmark/compressors-benchmarks.cu @@ -0,0 +1,248 @@ +// ============================================================================= +// benchmark_runner.cpp — single-file driver that mirrors test-scripts/run.sh +// +// * Builds with any C++17+ host compiler + NVCC. +// * Host-only code (filesystem, main, etc.) is hidden from the device pass +// via `#if !defined(__CUDA_ARCH__)` guards, so NVCC no longer chokes. +// +// Compile example: +// g++ -std=c++17 benchmark_runner.cpp -o compressors-benchmarks $(CUDA_LIBS) ... +// ============================================================================= +#include "engine/enums.cuh" +#include "flsgpu/consts.cuh" +#include "generator/generate_binaries.hpp" +#include "nvcomp/benchmark-compressors.cuh" +#include +#include +#include +#include +#if !defined(__CUDA_ARCH__) +#include // host-only +#endif +#include +#include +#include +#include +#include +#include + +// ----------------------------------------------------------------------------- +// Utility helpers (shared by host + device passes) +// ----------------------------------------------------------------------------- +inline std::string extract_filename(const std::string& path) { + const std::size_t pos = path.find_last_of('/'); + std::string fn = (pos == std::string::npos) ? path : path.substr(pos + 1); + const std::size_t ext = fn.find_last_of('.'); + return (ext == std::string::npos) ? fn : fn.substr(0, ext); +} + +struct CLIArgs { + enums::DataType data_type; + enums_nvcomp::ComparisonType comparison_type; + enums_nvcomp::CompressionType decompressor_enum; + std::string file_path; + int n_values; +}; + +inline CLIArgs parse_cli_args(int argc, char* argv[]) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] + << " " + " \n"; + throw std::invalid_argument("Incorrect CLI argument count"); + } + + int idx = 0; + CLIArgs args; + args.data_type = enums::string_to_data_type(argv[++idx]); + args.comparison_type = enums_nvcomp::string_to_comparison_type(argv[++idx]); + args.decompressor_enum = enums_nvcomp::string_to_compression_type(argv[++idx]); + args.file_path = argv[++idx]; + args.n_values = std::atoi(argv[++idx]) * consts::VALUES_PER_VECTOR; + return args; +} + +namespace data::arrays { +template +std::pair read_file_as(const std::string path, const size_t input_count) { + // Open file + std::ifstream inputFile(path, std::ios::binary | std::ios::ate); + if (!inputFile) { + throw std::invalid_argument("Could not open the specified file."); + } + // Get file size + const std::streamsize file_size = inputFile.tellg(); + inputFile.seekg(0, std::ios::beg); + + // Check file size to contain right type of data + bool file_size_is_multiple_of_T_size = static_cast(file_size) % static_cast(sizeof(T)) != 0; + if (file_size_is_multiple_of_T_size) { + throw std::invalid_argument("File size is incorrect, it is not a multiple of the type's size."); + } + + const size_t values_in_file = static_cast(file_size) / sizeof(T); + size_t count = input_count == 0 ? values_in_file : input_count; + count = count - (count % consts::VALUES_PER_VECTOR); + auto column = new T[count]; + + // Read either the file size, or the total number of values needed, + // whichever is smaller + const std::streamsize read_size = std::min(file_size, static_cast((count * sizeof(T)))); + if (!inputFile.read(reinterpret_cast(column), read_size)) { + throw std::invalid_argument("Failed to read file into column"); + } + + inputFile.close(); + + // Copy paste the values in file until the column is filled + if (values_in_file < count) { + size_t n_filled_values = values_in_file; + size_t n_empty_values_column = count - n_filled_values; + while (n_empty_values_column != 0) { + size_t n_values_to_copy = std::min(n_empty_values_column, values_in_file); + std::memcpy(column + n_filled_values, column, n_values_to_copy * sizeof(T)); + n_filled_values += n_values_to_copy; + n_empty_values_column -= n_values_to_copy; + } + } + + return std::make_pair(column, count); +} +} // namespace data::arrays + +// Template benchmark launcher (host-only code but harmless for device pass) +template +inline void execute_benchmark(const CLIArgs& args) { + auto [data, count] = data::arrays::read_file_as(args.file_path, args.n_values); + + uint32_t u = 0x4100f9db; // deterministic test value + const T value_to_search_for = *reinterpret_cast(&u); + + BenchmarkResult result; + for (int i = 0; i < 2; ++i) { + if (args.comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY && + args.decompressor_enum == enums_nvcomp::THRUST) { + result = benchmark_thrust(data, count, value_to_search_for); + } else if (args.decompressor_enum == enums_nvcomp::ALP || args.decompressor_enum == enums_nvcomp::GALP) { + result = benchmark_alp(args.comparison_type, args.decompressor_enum, data, count, value_to_search_for); + } else { + result = benchmark_hwc(args.comparison_type, args.decompressor_enum, data, count, value_to_search_for); + } + } + + result.log_result( + args.comparison_type, args.decompressor_enum, count * sizeof(T), extract_filename(args.file_path)); + + delete data; +} + +// ----------------------------------------------------------------------------- +// Main driver (host-only; skipped entirely for the device pass) +// ----------------------------------------------------------------------------- +#if !defined(__CUDA_ARCH__) +int main(int argc, char* argv[]) { + constexpr int VECTOR_COUNT = 25'600; // matches shell-script constant + + // ------------------------------------------------------------------------- + // Single-case CLI fallback (exactly 5 user args) + // ------------------------------------------------------------------------- + if (argc == 6) { + CLIArgs args = parse_cli_args(argc, argv); + if (args.data_type == enums::DataType::F32) + execute_benchmark(args); + else if (args.data_type == enums::DataType::F64) + execute_benchmark(args); + else + throw std::invalid_argument("Unsupported data type"); + return 0; + } + + // ------------------------------------------------------------------------- + // (0) Generate new binaries + discover directories + // ------------------------------------------------------------------------- + const size_t TOTAL = 25'600 * 1'024; + const size_t HEAD = 0; + + const std::filesystem::path floats_dir = FLS_GALP_SOURCE_DIR "/data/floats"; + const std::filesystem::path doubles_dir = FLS_GALP_SOURCE_DIR "/data/doubles"; + + bin::GenResult gen = bin::generate_write_and_scan(floats_dir, doubles_dir, TOTAL, HEAD); + + std::cout << "First " << HEAD << " values:\n"; + for (double v : gen.head) + std::cout << v << '\n'; + + if (gen.float_files.empty() && gen.double_files.empty()) { + std::cerr << "No .bin test files found. Populate data/floats and data/doubles.\n"; + return 1; + } + + // ------------------------------------------------------------------------- + // Benchmark matrix settings (mirrors test-scripts/run.sh) + // ------------------------------------------------------------------------- + const std::vector COMPARISONS = {enums_nvcomp::ComparisonType::DECOMPRESSION, + enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY}; + + const std::vector COMPRESSORS = { + // + // enums_nvcomp::ALP, + enums_nvcomp::GALP, + enums_nvcomp::BITCOMP, + enums_nvcomp::BITCOMP_SPARSE, + enums_nvcomp::LZ4, + enums_nvcomp::ZSTD, + enums_nvcomp::DEFLATE, + enums_nvcomp::GDEFLATE, + enums_nvcomp::SNAPPY + // + }; + + // CSV header + std::printf("comparison_type,compression_type,data_name,found_value," + "n_bytes,execution_time_ms,compression_ratio\n"); + + // ------------------------------------------------------------------------- + // (1) Full matrix — Floats (F32) + // ------------------------------------------------------------------------- + for (auto cmp : COMPARISONS) + for (auto decomp : COMPRESSORS) + for (const auto& path : gen.float_files) { + CLIArgs args {enums::DataType::F32, cmp, decomp, path, VECTOR_COUNT * consts::VALUES_PER_VECTOR}; + execute_benchmark(args); + } + + // ------------------------------------------------------------------------- + // (2) Full matrix — Doubles (F64) + // ------------------------------------------------------------------------- + for (auto cmp : COMPARISONS) + for (auto decomp : COMPRESSORS) + for (const auto& path : gen.double_files) { + CLIArgs args {enums::DataType::F64, cmp, decomp, path, VECTOR_COUNT * consts::VALUES_PER_VECTOR}; + execute_benchmark(args); + } + + // ------------------------------------------------------------------------- + // (3) Thrust path — only DECOMPRESSION_QUERY + // ------------------------------------------------------------------------- + auto run_thrust = [&](enums::DataType dtype, const std::vector& files) { + for (const auto& path : files) { + CLIArgs args; + args.data_type = dtype; + args.comparison_type = enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY; + args.decompressor_enum = enums_nvcomp::THRUST; + args.file_path = path; + args.n_values = VECTOR_COUNT * consts::VALUES_PER_VECTOR; + + if (dtype == enums::DataType::F32) + execute_benchmark(args); + else + execute_benchmark(args); + } + }; + + run_thrust(enums::DataType::F32, gen.float_files); + run_thrust(enums::DataType::F64, gen.double_files); + + return 0; +} +#endif // !__CUDA_ARCH__ diff --git a/galp/benchmark/generated-bindings/alp-double-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alp-double-decompress_column-bindings.cu new file mode 100644 index 00000000..382d0a2d --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-double-decompress_column-bindings.cu @@ -0,0 +1,105 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +double* decompress_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-double-query_column-bindings.cu b/galp/benchmark/generated-bindings/alp-double-query_column-bindings.cu new file mode 100644 index 00000000..45690a79 --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-double-query_column-bindings.cu @@ -0,0 +1,106 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-double-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alp-double-query_multi_column-bindings.cu new file mode 100644 index 00000000..aa73e73a --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-double-query_multi_column-bindings.cu @@ -0,0 +1,50 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-float-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alp-float-decompress_column-bindings.cu new file mode 100644 index 00000000..83973c9b --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-float-decompress_column-bindings.cu @@ -0,0 +1,146 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +float* decompress_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-float-query_column-bindings.cu b/galp/benchmark/generated-bindings/alp-float-query_column-bindings.cu new file mode 100644 index 00000000..a815cbf8 --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-float-query_column-bindings.cu @@ -0,0 +1,147 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-float-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alp-float-query_multi_column-bindings.cu new file mode 100644 index 00000000..8da7ec3d --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-float-query_multi_column-bindings.cu @@ -0,0 +1,65 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-double-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-double-decompress_column-bindings.cu new file mode 100644 index 00000000..318e9eb4 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-double-decompress_column-bindings.cu @@ -0,0 +1,134 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +double* decompress_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-double-query_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-double-query_column-bindings.cu new file mode 100644 index 00000000..17212ea4 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-double-query_column-bindings.cu @@ -0,0 +1,135 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-double-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-double-query_multi_column-bindings.cu new file mode 100644 index 00000000..f535dcd9 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-double-query_multi_column-bindings.cu @@ -0,0 +1,135 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>( + const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-float-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-float-decompress_column-bindings.cu new file mode 100644 index 00000000..da901b77 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-float-decompress_column-bindings.cu @@ -0,0 +1,189 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +float* decompress_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-float-query_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-float-query_column-bindings.cu new file mode 100644 index 00000000..2a841bfa --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-float-query_column-bindings.cu @@ -0,0 +1,190 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-float-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-float-query_multi_column-bindings.cu new file mode 100644 index 00000000..fb160bb8 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-float-query_multi_column-bindings.cu @@ -0,0 +1,190 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>( + const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint32_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint32_t-decompress_column-bindings.cu new file mode 100644 index 00000000..b5e91439 --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint32_t-decompress_column-bindings.cu @@ -0,0 +1,85 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include +namespace bindings { + +template <> +uint32_t* +decompress_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 32, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint32_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint32_t-query_column-bindings.cu new file mode 100644 index 00000000..8188cd0c --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint32_t-query_column-bindings.cu @@ -0,0 +1,86 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 32, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint64_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint64_t-decompress_column-bindings.cu new file mode 100644 index 00000000..2b65bafe --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint64_t-decompress_column-bindings.cu @@ -0,0 +1,74 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +uint64_t* +decompress_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint64_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint64_t-query_column-bindings.cu new file mode 100644 index 00000000..c5bbe035 --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint64_t-query_column-bindings.cu @@ -0,0 +1,74 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-compute_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-compute_column-bindings.cu new file mode 100644 index 00000000..1429ca14 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-compute_column-bindings.cu @@ -0,0 +1,91 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool compute_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in compute_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-decompress_column-bindings.cu new file mode 100644 index 00000000..f80eb85a --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-decompress_column-bindings.cu @@ -0,0 +1,86 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +uint32_t* +decompress_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-query_column-bindings.cu new file mode 100644 index 00000000..93219940 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-query_column-bindings.cu @@ -0,0 +1,86 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu new file mode 100644 index 00000000..e3e6a4c1 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu @@ -0,0 +1,60 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-compute_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-compute_column-bindings.cu new file mode 100644 index 00000000..201d8b83 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-compute_column-bindings.cu @@ -0,0 +1,78 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool compute_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in compute_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-decompress_column-bindings.cu new file mode 100644 index 00000000..e44c1f8c --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-decompress_column-bindings.cu @@ -0,0 +1,74 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +uint64_t* +decompress_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-query_column-bindings.cu new file mode 100644 index 00000000..d1b237dc --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-query_column-bindings.cu @@ -0,0 +1,73 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu new file mode 100644 index 00000000..3ec5c427 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu @@ -0,0 +1,48 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/include/generated-bindings/kernel-bindings.cuh b/galp/benchmark/include/generated-bindings/kernel-bindings.cuh new file mode 100644 index 00000000..4ba5b5f3 --- /dev/null +++ b/galp/benchmark/include/generated-bindings/kernel-bindings.cuh @@ -0,0 +1,211 @@ +#ifndef GENERATED_KERNEL_BINDINGS_CUH +#define GENERATED_KERNEL_BINDINGS_CUH + +#include "engine/enums.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include + +namespace bindings { + +template +T* decompress_column(const ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); + +template +bool query_column(const ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const T magic_value, + const uint32_t n_samples); + +template +bool compute_column(const ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples); + +template +bool query_multi_column(ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const T magic_value, + const uint32_t n_samples); + +uint32_t* decompress_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +uint64_t* decompress_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +uint32_t* decompress_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +uint64_t* decompress_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +float* decompress_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +double* decompress_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +float* decompress_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +double* decompress_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); + +bool query_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); + +bool compute_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples); +bool compute_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples); + +bool query_multi_column(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); + +} // namespace bindings + +#endif // GENERATED_KERNEL_BINDINGS_CUH diff --git a/galp/benchmark/include/generated-bindings/multi-column-device-kernels.cuh b/galp/benchmark/include/generated-bindings/multi-column-device-kernels.cuh new file mode 100644 index 00000000..6d1b8559 --- /dev/null +++ b/galp/benchmark/include/generated-bindings/multi-column-device-kernels.cuh @@ -0,0 +1,527 @@ +#ifndef MULTI_COLUMN_DEVICE_KERNELS_CUH +#define MULTI_COLUMN_DEVICE_KERNELS_CUH + +#include "engine/device-utils.cuh" +#include "flsgpu/flsgpu-api.cuh" + +namespace multi_column { + +template +__global__ void query_multi_column(const ColumnT column_0, const T value, bool* out) { + constexpr int32_t N_COLS = 1; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 1]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, const ColumnT column_1, const T value, bool* out) { + constexpr int32_t N_COLS = 2; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 2]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void +query_multi_column(const ColumnT column_0, const ColumnT column_1, const ColumnT column_2, const T value, bool* out) { + constexpr int32_t N_COLS = 3; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 3]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const T value, + bool* out) { + constexpr int32_t N_COLS = 4; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 4]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const T value, + bool* out) { + constexpr int32_t N_COLS = 5; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 5]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const T value, + bool* out) { + constexpr int32_t N_COLS = 6; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 6]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const T value, + bool* out) { + constexpr int32_t N_COLS = 7; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 7]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const ColumnT column_7, + const T value, + bool* out) { + constexpr int32_t N_COLS = 8; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 8]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + DecompressorT decompressor_7 = DecompressorT(column_7, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_7.unpack_next_into(registers + 7 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const ColumnT column_7, + const ColumnT column_8, + const T value, + bool* out) { + constexpr int32_t N_COLS = 9; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 9]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + DecompressorT decompressor_7 = DecompressorT(column_7, vector_index, lane); + DecompressorT decompressor_8 = DecompressorT(column_8, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_7.unpack_next_into(registers + 7 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_8.unpack_next_into(registers + 8 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const ColumnT column_7, + const ColumnT column_8, + const ColumnT column_9, + const T value, + bool* out) { + constexpr int32_t N_COLS = 10; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 10]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + DecompressorT decompressor_7 = DecompressorT(column_7, vector_index, lane); + DecompressorT decompressor_8 = DecompressorT(column_8, vector_index, lane); + DecompressorT decompressor_9 = DecompressorT(column_9, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_7.unpack_next_into(registers + 7 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_8.unpack_next_into(registers + 8 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_9.unpack_next_into(registers + 9 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +} // namespace multi_column +#endif // MULTI_COLUMN_DEVICE_KERNELS_CUH diff --git a/galp/benchmark/micro-benchmarks.cu b/galp/benchmark/micro-benchmarks.cu new file mode 100644 index 00000000..ccf7c656 --- /dev/null +++ b/galp/benchmark/micro-benchmarks.cu @@ -0,0 +1,261 @@ +#include "engine/data.cuh" +#include "engine/enums.cuh" +#include "engine/verification.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include +#include +#include +#include +#include +#include +#include + +struct ProgramParameters { + enums::DataType data_type; + enums::Kernel kernel; + uint32_t unpack_n_vecs; + uint32_t unpack_n_vals; + enums::Unpacker unpacker; + enums::Patcher patcher; + data::ValueRange bit_width_range; + data::ValueRange ec_range; + size_t n_values; + uint32_t n_samples; + enums::Print print_option; +}; + +struct CLIArgs { + std::string data_type; + std::string kernel; + uint32_t unpack_n_vecs; + uint32_t unpack_n_vals; + std::string patcher; + std::string unpacker; + vbw_t start_vbw; + vbw_t end_vbw; + uint16_t start_ec; + uint16_t end_ec; + size_t n_vecs; + uint32_t n_samples; + uint32_t print_debug; + + CLIArgs(const int argc, char** argv) { + constexpr int32_t CORRECT_ARG_COUNT = 14; + if (argc != CORRECT_ARG_COUNT) { + throw std::invalid_argument("Wrong arg count.\n"); + } + + int32_t argcounter = 0; + data_type = argv[++argcounter]; + kernel = argv[++argcounter]; + unpack_n_vecs = std::stoul(argv[++argcounter]); + unpack_n_vals = std::stoul(argv[++argcounter]); + unpacker = argv[++argcounter]; + patcher = argv[++argcounter]; + start_vbw = std::stoul(argv[++argcounter]); + end_vbw = std::stoul(argv[++argcounter]); + start_ec = std::stoul(argv[++argcounter]); + end_ec = std::stoul(argv[++argcounter]); + n_vecs = std::stoul(argv[++argcounter]); + n_samples = std::stoul(argv[++argcounter]); + print_debug = std::stoul(argv[++argcounter]); + } + + ProgramParameters parse() { + return ProgramParameters { + enums::string_to_data_type(data_type), + enums::string_to_kernel(kernel), + unpack_n_vecs, + unpack_n_vals, + enums::string_to_unpacker(unpacker), + enums::string_to_patcher(patcher), + data::ValueRange(start_vbw, end_vbw), + data::ValueRange(start_ec, end_ec), + n_vecs * consts::VALUES_PER_VECTOR, + n_samples, + static_cast(print_debug), + }; + } + +private: +}; + +template +verification::ExecutionResult decompress_column(const ColumnT column, const ProgramParameters params) { + auto column_device = column.copy_to_device(); + const T* out = bindings::decompress_column( + column_device, params.unpack_n_vecs, params.unpack_n_vals, params.unpacker, params.patcher, params.n_samples); + flsgpu::host::free_column(column_device); + + const T* correct_out = data::bindings::decompress(column); + auto result = verification::compare_data(correct_out, out, params.n_values); + delete correct_out; + delete out; + return result; +} + +template +verification::ExecutionResult +query_column(const ColumnT column, const ProgramParameters params, const bool query_result, const T magic_value) { + auto column_device = column.copy_to_device(); + const bool answer = bindings::query_column(column_device, + params.unpack_n_vecs, + params.unpack_n_vals, + params.unpacker, + params.patcher, + magic_value, + params.n_samples); + flsgpu::host::free_column(column_device); + + // Weird hack to avoid refactor_ + T a = query_result ? 1.0 : 0.0; + T b = answer ? 1.0 : 0.0; + + return verification::compare_data(&a, &b, 1); +} + +template +verification::ExecutionResult +query_multi_column(const ColumnT column, const ProgramParameters params, const bool query_result, const T magic_value) { + const bool answer = bindings::query_multi_column(column, + params.unpack_n_vecs, + params.unpack_n_vals, + params.unpacker, + params.patcher, + magic_value, + params.n_samples); + + // Weird hack to avoid refactor_ + T a = query_result ? 1.0 : 0.0; + T b = answer ? 1.0 : 0.0; + + return verification::compare_data(&a, &b, 1); +} + +template +verification::ExecutionResult +execute_kernel(const ColumnT column, const ProgramParameters params, const bool query_result, const T magic_value) { + if (params.kernel == enums::Kernel::Decompress) { + return decompress_column(column, params); + } else if (params.kernel == enums::Kernel::Query) { + return query_column(column, params, query_result, magic_value); + } else if (params.kernel == enums::Kernel::QueryMultiColumn) { + return query_multi_column(column, params, query_result, magic_value); + } else { + throw std::invalid_argument("Kernel not implemented yet.\n"); + } +} + +template +std::vector> execute_ffor(const ProgramParameters params) { + using UINT_T = typename utils::same_width_uint::type; + auto results = std::vector>(); + + for (vbw_t vbw {params.bit_width_range.min}; vbw <= params.bit_width_range.max; ++vbw) { + auto vbw_range = data::ValueRange(vbw); + if (params.kernel == enums::Kernel::QueryMultiColumn) { + vbw_range = params.bit_width_range; + } + bool query_result = false; + T magic_value = consts::as::MAGIC_NUMBER; + flsgpu::host::FFORColumn column; + + if (params.kernel == enums::Kernel::Query) { + auto [_query_result, _column] = + data::columns::generate_binary_ffor_column(params.n_values, vbw_range, params.unpack_n_vecs); + query_result = _query_result; + column = _column; + } else { + column = data::columns::generate_random_ffor_column( + params.n_values, vbw_range, data::ValueRange(0, 100), params.unpack_n_vecs); + } + + if (params.kernel == enums::Kernel::QueryMultiColumn) { + // We do not want query multicolumn to ever find a full lane of the + // value to query to limit write bandwidth + magic_value = std::numeric_limits::max(); + } + + results.push_back(execute_kernel>(column, params, query_result, magic_value)); + + flsgpu::host::free_column(column); + + if (params.kernel == enums::Kernel::QueryMultiColumn) { + break; + } + } + + return results; +} + +template +std::vector> execute_alp(const ProgramParameters params) { + using UINT_T = typename utils::same_width_uint::type; + auto results = std::vector>(); + + // for (vbw_t vbw{params.bit_width_range.min}; vbw <= + // params.bit_width_range.max; ++vbw) { + { + bool query_result = false; + T magic_value = consts::as::MAGIC_NUMBER; + + auto column = data::columns::generate_alp_column( + params.n_values, params.bit_width_range, data::ValueRange(0), params.unpack_n_vecs); + for (uint16_t ec {params.ec_range.min}; ec <= params.ec_range.max; ++ec) { + column = data::columns::modify_alp_exception_count(column, ec); + + if (params.kernel == enums::Kernel::Query) { + auto [_query_result, _magic_value] = + data::columns::get_value_to_query>(column); + query_result = _query_result; + magic_value = _magic_value; + } + + if (params.patcher == enums::Patcher::Dummy || params.patcher == enums::Patcher::Stateless || + params.patcher == enums::Patcher::Stateful) { + results.push_back( + execute_kernel>(column, params, query_result, magic_value)); + } else { + auto column_extended = column.create_extended_column(); + + results.push_back(execute_kernel>( + column_extended, params, query_result, magic_value)); + + flsgpu::host::free_column(column_extended); + } + } + + flsgpu::host::free_column(column); + } + + return results; +} + +int main(int argc, char** argv) { + CLIArgs args(argc, argv); + ProgramParameters params = args.parse(); + + int32_t exit_code = 0; + bool print_debug = params.print_option != enums::Print::PrintNothing; + switch (params.data_type) { + case enums::DataType::U32: + exit_code = verification::process_results(execute_ffor(params), print_debug); + break; + case enums::DataType::U64: + exit_code = verification::process_results(execute_ffor(params), print_debug); + break; + case enums::DataType::F32: + exit_code = verification::process_results(execute_alp(params), print_debug); + break; + case enums::DataType::F64: + exit_code = verification::process_results(execute_alp(params), print_debug); + break; + } + + if (params.print_option == enums::Print::PrintDebugExit0) { + exit(0); + } + + exit(exit_code); +} diff --git a/galp/benchmark/nvcomp/benchmark-compressors.cu b/galp/benchmark/nvcomp/benchmark-compressors.cu new file mode 100644 index 00000000..cb6623f0 --- /dev/null +++ b/galp/benchmark/nvcomp/benchmark-compressors.cu @@ -0,0 +1,211 @@ +#include "alp/alp-bindings.cuh" +#include "engine/data.cuh" +#include "engine/device-utils.cuh" +#include "engine/kernels.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include "nvcomp/benchmark-compressors.cuh" +#include "nvcomp/nvcomp-compressors.cuh" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template +using ALPDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>; + +template +using ALPExtendedDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>; + +template +BenchmarkResult benchmark_thrust(const T* input, const size_t value_count, const T value_to_search_for) { + CudaStopwatch stopwatch = CudaStopwatch(); + double execution_time_ms; + + thrust::device_vector d_vec(input, input + value_count); + stopwatch.start(); + bool result = thrust::any_of(thrust::device, d_vec.begin(), d_vec.end(), is_equal_to(value_to_search_for)); + execution_time_ms = stopwatch.stop(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + + return BenchmarkResult {result, execution_time_ms, 1.0}; +} + +template +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const T* input, + const flsgpu::host::ALPColumn column, + const T value_to_search_for) { + bool result = false; + GPUArray d_query_result(1, &result); + GPUArray d_decompression_result(column.get_n_values()); + + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + CudaStopwatch stopwatch = CudaStopwatch(); + double execution_time_ms; + double compression_ratio; + + switch (decompressor_enum) { + case enums_nvcomp::ALP: { + flsgpu::device::ALPColumn d_column = column.copy_to_device(); + stopwatch.start(); + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + kernels::device::decompress_column, + flsgpu::device::ALPColumn> + <<>>(d_column, d_decompression_result.get()); + } else { + kernels::device:: + query_column, flsgpu::device::ALPColumn> + <<>>( + d_column, d_query_result.get(), value_to_search_for); + } + execution_time_ms = stopwatch.stop(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + flsgpu::host::free_column(d_column); + compression_ratio = column.get_compression_ratio(); + } break; + + case enums_nvcomp::GALP: { + flsgpu::host::ALPExtendedColumn column_extended = column.create_extended_column(); + flsgpu::device::ALPExtendedColumn d_column = column_extended.copy_to_device(); + stopwatch.start(); + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + kernels::device::decompress_column, + flsgpu::device::ALPExtendedColumn> + <<>>(d_column, d_decompression_result.get()); + } else { + kernels::device::query_column, + flsgpu::device::ALPExtendedColumn> + <<>>( + d_column, d_query_result.get(), value_to_search_for); + } + execution_time_ms = stopwatch.stop(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + compression_ratio = column_extended.get_compression_ratio(); + flsgpu::host::free_column(column_extended); + flsgpu::host::free_column(d_column); + } break; + default: + throw std::invalid_argument("Could not parse decompressor enum for alp\n"); + } + + bool kernel_successful = false; + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + GPUArray d_input(column.get_n_values(), input); + + kernel_successful = + check_if_device_buffers_are_equal(d_decompression_result.get(), d_input.get(), column.get_n_values()); + } + + return BenchmarkResult {kernel_successful, execution_time_ms, compression_ratio}; +} + +template +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const T* input, + const size_t value_count, + const T value_to_search_for) { + flsgpu::host::ALPColumn column = alp::encode(input, value_count, true); + auto result = benchmark_alp(comparison_type, decompressor_enum, input, column, value_to_search_for); + flsgpu::host::free_column(column); + + return result; +} + +template +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const T* input, + const size_t value_count, + const T value_to_search_for) { + size_t size_in_bytes = value_count * sizeof(T); + GPUArray d_input_buffer(size_in_bytes, reinterpret_cast(input)); + hwc::Compressor compressor(compression_type); + hwc::CompressedBuffer d_compressed_buffer = compressor.compress(d_input_buffer.get(), size_in_bytes); + + GPUArray d_query_result(1); + double compression_ratio = d_compressed_buffer.get_compression_ratio(); + + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, utils::get_n_vecs_from_size(value_count)); + + CudaStopwatch stopwatch = CudaStopwatch(); + uint8_t* d_output_buffer = compressor.decompress(d_compressed_buffer, stopwatch); + + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY) { + hwc::DummyColumn d_column {reinterpret_cast(d_output_buffer), value_count}; + kernels::device::query_column, hwc::DummyColumn> + <<>>(d_column, d_query_result.get(), value_to_search_for); + } + + double execution_time_ms = stopwatch.get_result(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + + bool kernel_successful = false; + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + kernel_successful = + check_if_device_buffers_are_equal(d_output_buffer, d_input_buffer.get(), size_in_bytes); + } else if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY) { + d_query_result.copy_to_host(&kernel_successful); + } + + CUDA_SAFE_CALL(cudaFree(d_output_buffer)); + d_compressed_buffer.free(); + compressor.free(); + + return BenchmarkResult {kernel_successful, execution_time_ms, compression_ratio}; +} + +template BenchmarkResult +benchmark_thrust(const float* input, const size_t value_count, const float value_to_search_for); +template BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const float* input, + const size_t value_count, + const float value_to_search_for); +template BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const float* input, + const size_t value_count, + const float value_to_search_for); + +template BenchmarkResult +benchmark_thrust(const double* input, const size_t value_count, const double value_to_search_for); +template BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const double* input, + const size_t value_count, + const double value_to_search_for); +template BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const double* input, + const size_t value_count, + const double value_to_search_for); diff --git a/galp/benchmark/nvcomp/nvcomp-compressors.cu b/galp/benchmark/nvcomp/nvcomp-compressors.cu new file mode 100644 index 00000000..f21a14fd --- /dev/null +++ b/galp/benchmark/nvcomp/nvcomp-compressors.cu @@ -0,0 +1,80 @@ +#include "nvcomp/nvcomp-compressors.cuh" +#include +#include +#include +#include +#include + +namespace hwc { +nvcompCompressionManager* get_compressor_manager(const enums_nvcomp::CompressionType compression_type, + const nvcompType_t data_type, + const size_t chunk_size) { + switch (compression_type) { + case enums_nvcomp::CompressionType::BITCOMP: { + nvcompBatchedBitcompOpts_t format_opts {0, data_type}; + return new nvcomp::BitcompManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::BITCOMP_SPARSE: { + nvcompBatchedBitcompOpts_t format_opts {1, data_type}; + return new nvcomp::BitcompManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::LZ4: { + nvcompBatchedLZ4Opts_t format_opts {data_type}; + return new nvcomp::LZ4Manager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::ZSTD: { + nvcompBatchedZstdOpts_t format_opts {data_type}; + return new nvcomp::ZstdManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::DEFLATE: { + nvcompBatchedDeflateOpts_t format_opts {data_type}; + return new nvcomp::DeflateManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::GDEFLATE: { + nvcompBatchedGdeflateOpts_t format_opts {data_type}; + return new nvcomp::GdeflateManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::SNAPPY: { + nvcompBatchedSnappyOpts_t format_opts {data_type}; + return new nvcomp::SnappyManager {chunk_size, format_opts}; + } break; + default: + throw std::invalid_argument("Compression type is not supported"); + } +} + +__global__ void +check_buffer_equality(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length, bool* out) { + constexpr int32_t N_ELEMENTS_PER_THREAD = 4 * 32; + size_t index = N_ELEMENTS_PER_THREAD * (threadIdx.x + blockIdx.x * blockDim.x); + + bool buffers_are_equal = true; + +#pragma unroll + for (int32_t i {0}; i < N_ELEMENTS_PER_THREAD; ++i) { + if (index + i < length) { + buffers_are_equal &= buffer_a[index + i] == buffer_b[index + i]; + } + } + + if (!buffers_are_equal) { + *out = false; + } +} + +bool compare_d_buffers(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length) { + bool result = true; + bool* d_result; + CUDA_SAFE_CALL(cudaMalloc(&d_result, sizeof(bool) * 1)); + CUDA_SAFE_CALL(cudaMemcpy(d_result, &result, sizeof(bool) * 1, cudaMemcpyHostToDevice)); + + const int n_blocks = (length + (1024 * 128) - 1) / (1024 * 128); + check_buffer_equality<<>>(buffer_a, buffer_b, length, d_result); + + CUDA_SAFE_CALL(cudaMemcpy(&result, d_result, sizeof(bool) * 1, cudaMemcpyDeviceToHost)); + CUDA_SAFE_CALL(cudaFree(d_result)); + + return result; +} + +} // namespace hwc diff --git a/galp/code-generators/generate-kernel-bindings.py b/galp/code-generators/generate-kernel-bindings.py new file mode 100755 index 00000000..6ebd142f --- /dev/null +++ b/galp/code-generators/generate-kernel-bindings.py @@ -0,0 +1,429 @@ +#!/usr/bin/python3 + +import os +import sys + +import argparse +import logging + +GENERATED_BINDINGS_DIR = "./src/generated-bindings/" + +FILE_HEADER = """ +#include + +#include "kernel-bindings.cuh" +#include "../engine/kernels.cuh" +#include "../engine/multi-column-host-kernels.cuh" + +namespace bindings{ +""" + +FILE_FOOTER = """ +} +""" + + +DATA_TYPES = [ + "uint32_t", + "uint64_t", + "float", + "double", +] + +FUNCTIONS = [ + "decompress_column", + "query_column", + "compute_column", + "query_multi_column", +] + +ENCODINGS = [ + "BP", + "FFOR", + "ALP", + "ALPExtended", +] + +UNPACKERS = [ + "Dummy", + "OldFls", + #"SwitchCase", + #"Stateless", + #"StatelessBranchless", + #"StatefulCache", + #"StatefulLocal1", + #"StatefulLocal2", + #"StatefulLocal4", + #"StatefulShared1", + #"StatefulShared2", + #"StatefulShared4", + #"StatefulRegister1", + #"StatefulRegister2", + #"StatefulRegister4", + #"StatefulRegisterBranchless1", + #"StatefulRegisterBranchless2", + #"StatefulRegisterBranchless4", + "StatefulBranchless", +] +MULTI_COLUMN_UNPACKERS = [ + UNPACKERS[1], + UNPACKERS[2], +] + +PATCHERS = [ + "None", + "Dummy", + "Stateless", + "Stateful", + "Naive", + "NaiveBranchless", + "PrefetchAll", + "PrefetchAllBranchless", +] +MULTI_COLUMN_PATCHERS = [ + PATCHERS[0], + PATCHERS[3], + PATCHERS[4], + PATCHERS[5], + PATCHERS[6], + PATCHERS[7], +] + + +def get_column_t( + encoding: str, data_type: str, function: str, for_decompressor: bool = False +) -> str: + column_t = f"BPColumn<{data_type}>" + if "FFOR" in encoding: + column_t = f"FFORColumn<{data_type}>" + elif "ALPExtended" in encoding: + column_t = f"ALPExtendedColumn<{data_type}>" + elif "ALP" in encoding: + column_t = f"ALPColumn<{data_type}>" + return ( + "flsgpu::device::" + if function != "query_multi_column" or for_decompressor + else "flsgpu::host::" + ) + column_t + + +def get_decompressor_type( + encoding: str, + data_type: str, + function: str, + unpacker: str, + patcher: str, + n_vec: int, + n_val: int, +) -> str: + column_t = get_column_t(encoding, data_type, function, True) + functor = f"BPFunctor<{data_type}>" + patcher_t = f"" + decompressor_t = f"{encoding}Decompressor" + if "FFOR" in encoding: + functor = f"FFORFunctor<{data_type}, {n_vec}>" + elif "ALPExtended" in encoding: + functor = f"ALPFunctor<{data_type}, {n_vec}>" + patcher_t = f"flsgpu::device::{patcher}ALPExceptionPatcher<{data_type}, {n_vec}, {n_val}>," + decompressor_t = "ALPDecompressor" + elif "ALP" in encoding: + functor = f"ALPFunctor<{data_type}, {n_vec}>" + patcher_t = f"flsgpu::device::{patcher}ALPExceptionPatcher<{data_type}, {n_vec}, {n_val}>," + + loader_t = "" + if "Stateful" in unpacker and "StatefulBranchless" not in unpacker: + loader_t = ", flsgpu::device::" + if "Cache" in unpacker: + loader_t += f"CacheLoader<{data_type}, {n_vec}>" + elif "Local" in unpacker: + loader_t += f"LocalMemoryLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + elif "Shared" in unpacker: + loader_t += f"SharedMemoryLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + elif "RegisterBranchless" in unpacker: + loader_t += ( + f"RegisterBranchlessLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + ) + elif "Register" in unpacker: + loader_t += f"RegisterLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + unpacker = "Stateful" + + unpacker_t = f"flsgpu::device::BitUnpacker{unpacker}<{data_type}, {n_vec}, {n_val}, flsgpu::device::{functor} {loader_t}>" + + return f"flsgpu::device::{decompressor_t}<{data_type}, {n_vec}, {unpacker_t}, {patcher_t} {column_t}>" + + +def get_if_statement( + encoding: str, + data_type: str, + function: str, + n_vec: int, + n_val: int, + unpacker: str, + patcher: str, + is_query_column: bool = False, + n_columns: int | None = None, + n_repetitions: int | None = None, +) -> str: + assert data_type in DATA_TYPES + assert function in FUNCTIONS + assert n_vec in [1, 2, 4, 8] + assert n_val in [1, 32] + assert unpacker in UNPACKERS + assert patcher in PATCHERS + + column_t = get_column_t(encoding, data_type, function) + decompressor_t = get_decompressor_type( + encoding, data_type, function, unpacker, patcher, n_vec, n_val + ) + extra_param = ( + "," + str(n_columns) + if n_columns + else ", magic_value" if is_query_column else "" + ) + return ( + f"if (unpack_n_vectors == {n_vec} && unpack_n_values == {n_val} && unpacker == enums::Unpacker::{unpacker} && patcher == enums::Patcher::{patcher} {'&& n_columns == ' + str(n_columns) if n_columns else ''}) " + + "{" # } + f"return kernels::host::{function}<{data_type}, {n_vec}, {n_val}, {decompressor_t}, {column_t} {',' + str(n_repetitions) if n_repetitions else ''}>(column {extra_param}, n_samples);" + "}" + ) + + +def get_function( + encoding: str, + data_type: str, + function: str, + return_type: str, + content: list[str], + is_query_column: bool = False, + is_multi_column: bool = False, + is_compute_column: bool = False, +) -> str: + assert not (is_multi_column and is_compute_column) + column_t = get_column_t(encoding, data_type, function) + return ( + f"template<> {return_type} {function}<{data_type},{column_t}>(const {column_t} column, const unsigned unpack_n_vectors, const unsigned unpack_n_values, const enums::Unpacker unpacker, const enums::Patcher patcher {', const ' + data_type + ' magic_value' if is_query_column or is_multi_column else ''}{', const unsigned n_repetitions' if is_compute_column else ''}, const uint32_t n_samples)" + + "{" + + "\n".join(content) + + f'throw std::invalid_argument("Could not find correct binding in {function} {encoding}<{data_type}>");' + + "}" + ) + + +def write_file( + file_name: str, + functions: list[str], +): + logging.info(f"Writing file {file_name}") + with open(os.path.join(GENERATED_BINDINGS_DIR, file_name), "w") as f: + f.write("\n".join([FILE_HEADER] + functions + [FILE_FOOTER])) + + +def get_if_statement_check_wrapper( + disable_unnecessary: bool, + encoding: str, + data_type: str, + function: str, + n_vec: int, + n_val: int, + unpacker: str, + patcher: str, + is_query_column: bool = False, + n_columns: int | None = None, + n_repetitions: int | None = None, +) -> str: + # du handling + is_necessary = ( + data_type == "float" + and function == "decompress_column" + and unpacker == "OldFls" + ) + + # OldFls handling + if unpacker == "OldFls": + n_val = 32 + + # Filters + unnessary_filter = disable_unnecessary and not is_necessary + switch_case_filter = unpacker == "SwitchCase" and ( + n_vec != 1 + or n_val != 1 + or "uint" not in data_type + or function == "compute_column" + ) + multi_column_filter = function == "query_multi_column" and ( + unpacker not in MULTI_COLUMN_UNPACKERS + or patcher not in MULTI_COLUMN_PATCHERS + or args.disable_multi_column + ) + old_fls_filter = unpacker == "OldFls" and ( + n_vec != 1 or data_type not in ["uint32_t", "float"] + ) + + is_filtered = ( + unnessary_filter or switch_case_filter or multi_column_filter or old_fls_filter + ) + if is_filtered: + return "" + + return get_if_statement( + encoding, + data_type, + function, + n_vec, + n_val, + unpacker, + patcher, + is_query_column, + n_columns, + n_repetitions, + ) + + +def main(args): + for encoding in ["BP", "FFOR"]: + for data_type in ["uint32_t", "uint64_t"]: + for binding, is_query_column in zip( + ["decompress_column", "query_column"], [False, True] + ): + write_file( + f"{encoding.lower()}-{data_type}-{binding}-bindings.cu", + [ + get_function( + encoding, + data_type, + binding, + "bool" if is_query_column else data_type + "*", + [ + get_if_statement_check_wrapper( + args.disable_unnecessary, + encoding, + data_type, + binding, + n_vec, + n_val, + unpacker, + "None", + is_query_column=is_query_column, + ) + for n_vec in [1, 4] + for n_val in [1] + for unpacker in UNPACKERS + ], + is_query_column=is_query_column, + ) + ], + ) + + for encoding in ["FFOR"]: + for data_type in ["uint32_t", "uint64_t"]: + for binding in ["query_multi_column", "compute_column"]: + is_compute_column = binding == "compute_column" + is_multi_column = binding == "query_multi_column" + write_file( + f"{encoding.lower()}-{data_type}-{binding}-bindings.cu", + [ + get_function( + encoding, + data_type, + binding, + "bool", + [ + get_if_statement_check_wrapper( + args.disable_unnecessary, + encoding, + data_type, + binding, + n_vec, + n_val, + unpacker, + "None", + n_repetitions=10 if is_compute_column else None, + is_query_column=is_multi_column, + ) + for n_vec in [1, 4] + for n_val in [1] + for unpacker in UNPACKERS + ], + is_multi_column=is_multi_column, + is_compute_column=is_compute_column, + ) + ], + ) + + for encoding, patchers_per_encoding in zip( + ["ALP", "ALPExtended"], [PATCHERS[1:4], PATCHERS[4:]] + ): + for data_type in ["float", "double"]: + for binding in ["decompress_column", "query_column", "query_multi_column"]: + is_query_column = binding == "query_column" + is_multi_column = binding == "query_multi_column" + write_file( + f"{encoding.lower()}-{data_type}-{binding}-bindings.cu", + [ + get_function( + encoding, + data_type, + binding, + ( + "bool" + if is_query_column or is_multi_column + else data_type + "*" + ), + [ + get_if_statement_check_wrapper( + args.disable_unnecessary, + encoding, + data_type, + binding, + n_vec, + n_val, + unpacker, + patcher, + is_query_column=is_query_column or is_multi_column, + n_repetitions=None, + ) + for n_vec in [1, 4] + for n_val in [1] + for unpacker in UNPACKERS[1:] + for patcher in patchers_per_encoding + ], + is_query_column=is_query_column, + is_multi_column=is_multi_column, + ) + ], + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog="program") + + parser.add_argument( + "-du", + "--disable-unnecessary", + type=bool, + default=False, + action=argparse.BooleanOptionalAction, + ) + parser.add_argument( + "-dmc", + "--disable-multi-column", + type=bool, + default=False, + action=argparse.BooleanOptionalAction, + ) + parser.add_argument( + "-ll", + "--logging-level", + type=int, + default=logging.INFO, + choices=[logging.CRITICAL, logging.ERROR, logging.INFO, logging.DEBUG], + help=f"logging level to use: {logging.CRITICAL}=CRITICAL, {logging.ERROR}=ERROR, {logging.INFO}=INFO, " + + f"{logging.DEBUG}=DEBUG, higher number means less output", + ) + + args = parser.parse_args() + logging.basicConfig(level=args.logging_level) # filename='program.log', + logging.info( + f"Started {os.path.basename(sys.argv[0])} with the following args: {args}" + ) + main(args) diff --git a/galp/code-generators/generate-multicolumn-kernels.py b/galp/code-generators/generate-multicolumn-kernels.py new file mode 100755 index 00000000..b60f97a1 --- /dev/null +++ b/galp/code-generators/generate-multicolumn-kernels.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +import os +import sys + +import argparse +import logging + +FILE_HEADER = """ +#include "../flsgpu/flsgpu-api.cuh" +#include "../engine/device-utils.cuh" + +#ifndef MULTI_COLUMN_DEVICE_KERNELS_CUH +#define MULTI_COLUMN_DEVICE_KERNELS_CUH + +namespace multi_column { +""" + +FILE_FOOTER = """ +} +#endif // MULTI_COLUMN_DEVICE_KERNELS_CUH +""" + +def generate_global_function(n_cols: int): + col_range = range(n_cols) + return "\n".join( + [ + "template ", + "__global__ void query_multi_column(" + + ",".join([f"const ColumnT column_{i}" for i in col_range]) + + ", const T value, bool *out) {", + f"constexpr int32_t N_COLS = {n_cols};", + "const auto mapping = VectorToWarpMapping();", + "const int32_t vector_index = mapping.get_vector_index();", + "const lane_t lane = mapping.get_lane();", + f"T registers[UNPACK_N_VALUES * UNPACK_N_VECS * {n_cols}];", + "bool all_columns_equal = true;", + "\n".join( + [ + f"DecompressorT decompressor_{i} = DecompressorT(column_{i}, vector_index, lane);" + for i in col_range + ] + ), + "for (si_t i{0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) {", + "\n".join( + [ + f"decompressor_{i}.unpack_next_into(registers + {i} * (UNPACK_N_VALUES * UNPACK_N_VECS));" + for i in col_range + ] + ), + "#pragma unroll", + "for (int c{1}; c < N_COLS; ++c) {", + "#pragma unroll", + "for (int va{0}; va < UNPACK_N_VALUES; ++va) {", + "#pragma unroll", + "for (int v{0}; v < UNPACK_N_VECS; ++v) {", + "all_columns_equal &= registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == registers[va + v * UNPACK_N_VALUES + (c-1) * UNPACK_N_VECS * UNPACK_N_VALUES];", + "}", + "}", + "}", + "#pragma unroll", + "for (int va{0}; va < UNPACK_N_VALUES; ++va) {", + "#pragma unroll", + "for (int v{0}; v < UNPACK_N_VECS; ++v) {", + "all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value;", + "}", + "}", + "}", + "", + "if (all_columns_equal) {", + "*out = true;", + "}}", + ] + ) + + +def main(args): + n_cols = 10 + generated_str = "\n\n".join( + [ + FILE_HEADER, + *[generate_global_function(n) for n in range(1, n_cols + 1)], + FILE_FOOTER, + ] + ) + + with open("src/generated-bindings/multi-column-device-kernels.cuh", "w") as file: + file.write(generated_str) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog="program") + + parser.add_argument( + "-ll", + "--logging-level", + type=int, + default=logging.INFO, + choices=[logging.CRITICAL, logging.ERROR, logging.INFO, logging.DEBUG], + help=f"logging level to use: {logging.CRITICAL}=CRITICAL, {logging.ERROR}=ERROR, {logging.INFO}=INFO, " + + f"{logging.DEBUG}=DEBUG, higher number means less output", + ) + + args = parser.parse_args() + logging.basicConfig(level=args.logging_level) # filename='program.log', + logging.info( + f"Started {os.path.basename(sys.argv[0])} with the following args: {args}" + ) + main(args) diff --git a/galp/src/CMakeLists.txt b/galp/src/CMakeLists.txt new file mode 100644 index 00000000..9592f512 --- /dev/null +++ b/galp/src/CMakeLists.txt @@ -0,0 +1,12 @@ +set(GALP_CORE_SOURCES + alp/alp-bindings.cu + engine/enums.cu +) +add_library(galp_core STATIC ${GALP_CORE_SOURCES}) +target_compile_options(galp_core PRIVATE $<$:-expt-relaxed-constexpr>) +target_include_directories(galp_core PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CUDAToolkit_INCLUDE_DIRS}) +target_link_libraries(galp_core PUBLIC CUDA::cudart nvcomp::nvcomp Thrust::Thrust FastLanes::core) +apply_warnings(galp_core) +set_target_properties(galp_core PROPERTIES CUDA_SEPARABLE_COMPILATION ON) \ No newline at end of file diff --git a/galp/src/alp/alp-bindings.cu b/galp/src/alp/alp-bindings.cu new file mode 100644 index 00000000..72b02a59 --- /dev/null +++ b/galp/src/alp/alp-bindings.cu @@ -0,0 +1,263 @@ +#include "alp.hpp" +#include "alp/alp-bindings.cuh" +#include +#include +#include +#include +#include +#include + +namespace alp { +constexpr int MAX_ATTEMPTS_TO_ENCODE = 10000; + +template +size_t get_bytes_overhead_size_per_alp_vector() { + return sizeof(uint8_t) + // bit_width + sizeof(uint8_t) + // factor-idx + sizeof(uint8_t) + // exponent-idx + sizeof(T); // ffor base + // + 32; // Overhead of vector offset in packed array + // + 32; // Overhead of vector offset in exception array +}; + +template +size_t get_bytes_overhead_size_per_alp_extended_vector() { + return get_bytes_overhead_size_per_alp_vector() + + sizeof(uint16_t) * utils::get_n_lanes(); // pos + offset per lane + // data parallel_format; +}; + +template +size_t get_bytes_vector_compressed_size_without_overhead(const uint8_t bit_width, const uint16_t exceptions_count) { + constexpr size_t line_size = utils::get_n_lanes() * sizeof(T); + constexpr size_t exception_value_size = sizeof(T); + constexpr size_t exception_position_size = sizeof(uint16_t); + + return bit_width * line_size + exceptions_count * (exception_value_size + exception_position_size); +} + +template +bool is_compressable(const T* input_array, const size_t count) { + T* sample_array = new T[count]; + state alpstate; + + bool is_possible = false; + for (int32_t attempts = 0; attempts < MAX_ATTEMPTS_TO_ENCODE; ++attempts) { + alp::encoder::init(input_array, count, sample_array, alpstate); + + if ((is_possible = alpstate.scheme == alp::Scheme::ALP)) { + break; + } + } + + delete[] sample_array; + return is_possible; +} + +template +state configure_alpstate(const T* input_array, const size_t n_values) { + T* sample_array = new T[n_values]; + state alpstate; + + bool successful_encoding = false; + for (int32_t attempts = 0; attempts < MAX_ATTEMPTS_TO_ENCODE; ++attempts) { + alp::encoder::init(input_array, n_values, sample_array, alpstate); + + if ((successful_encoding = alpstate.scheme == Scheme::ALP)) { + break; + } + } + if (!successful_encoding) { + throw alp::EncodingException(); + } + delete[] sample_array; + return alpstate; +} + +template +flsgpu::host::ALPColumn encode(const T* input_array, const size_t n_values, const bool print_compression_info) { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + + const size_t n_vecs = utils::get_n_vecs_from_size(n_values); + state alpstate = configure_alpstate(input_array, n_values); + + // Intermediate arrays + UINT_T* packed_array = new UINT_T[consts::VALUES_PER_VECTOR]; + T* exceptions = new T[consts::VALUES_PER_VECTOR]; + uint16_t* positions = new uint16_t[consts::VALUES_PER_VECTOR]; + + auto v_packed_array = std::vector(); + auto v_exceptions = std::vector(); + auto v_positions = std::vector(); + + // Final arrays + vbw_t* bit_widths = new vbw_t[n_vecs]; + size_t* vector_offsets = new size_t[n_vecs]; + UINT_T* bases = new UINT_T[n_vecs]; + uint8_t* factor_indices = new uint8_t[n_vecs]; + uint8_t* fraction_indices = new uint8_t[n_vecs]; + size_t* exceptions_offsets = new size_t[n_vecs]; + uint16_t* counts = new uint16_t[n_vecs]; + + size_t compressed_vector_sizes = 0; + INT_T* encoded_array = new INT_T[consts::VALUES_PER_VECTOR]; + size_t exceptions_offset = 0; + size_t vector_offset = 0; + + size_t bit_widths_sum = 0; + for (size_t vi {0}; vi < n_vecs; vi++) { + alp::encoder::encode(input_array, exceptions, positions, encoded_array, alpstate, nullptr); + alp::encoder::analyze_ffor(encoded_array, bit_widths[vi], reinterpret_cast(&bases[vi])); + + counts[vi] = alpstate.n_exceptions; + bit_widths_sum += bit_widths[vi]; + fastlanes::generated::ffor::fallback::scalar::ffor( + reinterpret_cast(encoded_array), packed_array, bit_widths[vi], &bases[vi]); + + input_array += consts::VALUES_PER_VECTOR; + size_t compressed_values_size = utils::get_compressed_vector_size(bit_widths[vi]); + + v_exceptions.insert(v_exceptions.end(), exceptions, exceptions + counts[vi]); + v_positions.insert(v_positions.end(), positions, positions + counts[vi]); + + v_packed_array.insert(v_packed_array.end(), packed_array, packed_array + compressed_values_size); + + factor_indices[vi] = alpstate.fac; + fraction_indices[vi] = alpstate.exp; + exceptions_offsets[vi] = exceptions_offset; + exceptions_offset += counts[vi]; + vector_offsets[vi] = vector_offset; + vector_offset += compressed_values_size; + + compressed_vector_sizes += + alp::get_bytes_vector_compressed_size_without_overhead(bit_widths[vi], counts[vi]); + } + delete[] encoded_array; + + size_t compressed_alp_bytes_size = + compressed_vector_sizes + n_vecs * alp::get_bytes_overhead_size_per_alp_vector(); + size_t compressed_alp_extended_bytes_size = + compressed_vector_sizes + n_vecs * alp::get_bytes_overhead_size_per_alp_extended_vector(); + + // Wrapping it up + delete[] packed_array; + delete[] exceptions; + delete[] positions; + + packed_array = new UINT_T[v_packed_array.size()]; + exceptions = new T[v_exceptions.size()]; + positions = new uint16_t[v_positions.size()]; + + std::memcpy(packed_array, v_packed_array.data(), v_packed_array.size() * sizeof(UINT_T)); + std::memcpy(exceptions, v_exceptions.data(), v_exceptions.size() * sizeof(T)); + std::memcpy(positions, v_positions.data(), v_positions.size() * sizeof(uint16_t)); + + if (print_compression_info) { + const size_t input_size = n_values * sizeof(T); + printf("ALP_COMPRESSION_PARAMETERS,%zu,%f,%f,%f,%f\n", + n_vecs, + static_cast(input_size) / static_cast(compressed_alp_bytes_size), + static_cast(input_size) / static_cast(compressed_alp_extended_bytes_size), + static_cast(bit_widths_sum) / static_cast(n_vecs), + static_cast(v_exceptions.size()) / static_cast(n_vecs)); + } + + return flsgpu::host::ALPColumn { + flsgpu::host::FFORColumn { + flsgpu::host::BPColumn { + n_values, + v_packed_array.size(), + packed_array, + bit_widths, + vector_offsets, + }, + bases, + }, + factor_indices, + fraction_indices, + v_exceptions.size(), + exceptions_offsets, + exceptions, + positions, + counts, + compressed_alp_bytes_size, + compressed_alp_extended_bytes_size, + }; +} + +template +T* decode(const flsgpu::host::ALPColumn column, T* output_array) { + const size_t n_vecs = utils::get_n_vecs_from_size(column.ffor.bp.n_values); + + T* c_output_array = output_array; + for (size_t vi {0}; vi < n_vecs; ++vi) { + generated::falp::fallback::scalar::falp(column.ffor.bp.packed_array + column.ffor.bp.vector_offsets[vi], + c_output_array, + column.ffor.bp.bit_widths[vi], + &column.ffor.bases[vi], + column.factor_indices[vi], + column.fraction_indices[vi]); + + alp::state alpstate; + alpstate.n_exceptions = column.counts[vi]; + + alp::decoder::patch_exceptions(c_output_array, + column.exceptions + column.exceptions_offsets[vi], + column.positions + column.exceptions_offsets[vi], + alpstate); + + c_output_array += consts::VALUES_PER_VECTOR; + } + + return output_array; +} + +template +T* decode(const flsgpu::host::ALPExtendedColumn column, T* output_array) { + constexpr unsigned N_LANES = utils::get_n_lanes(); + const size_t n_vecs = utils::get_n_vecs_from_size(column.ffor.bp.n_values); + + T* c_output_array = output_array; + for (size_t vi {0}; vi < n_vecs; ++vi) { + generated::falp::fallback::scalar::falp(column.ffor.bp.packed_array + column.ffor.bp.vector_offsets[vi], + c_output_array, + column.ffor.bp.bit_widths[vi], + &column.ffor.bases[vi], + column.factor_indices[vi], + column.fraction_indices[vi]); + + // Reconstruct total count + uint16_t count = 0; + for (size_t offset_count_i {0}; offset_count_i < N_LANES; ++offset_count_i) { + count += column.offsets_counts[vi * N_LANES + offset_count_i] >> 10; + } + + alp::state alpstate; + alpstate.n_exceptions = count; // fix me + + alp::decoder::patch_exceptions(c_output_array, + column.exceptions + column.exceptions_offsets[vi], + column.positions + column.exceptions_offsets[vi], + alpstate); + + c_output_array += consts::VALUES_PER_VECTOR; + } + + return output_array; +} + +template bool is_compressable(const float* input_array, const size_t n_values); +template bool is_compressable(const double* input_array, const size_t n_values); + +template flsgpu::host::ALPColumn +encode(const float* input_array, const size_t n_values, const bool print_compression_info); +template flsgpu::host::ALPColumn +encode(const double* input_array, const size_t n_values, const bool print_compression_info); + +template float* decode(const flsgpu::host::ALPColumn column, float* output_array); +template double* decode(const flsgpu::host::ALPColumn column, double* output_array); +template float* decode(const flsgpu::host::ALPExtendedColumn column, float* output_array); +template double* decode(const flsgpu::host::ALPExtendedColumn column, double* output_array); + +} // namespace alp diff --git a/galp/src/engine/enums.cu b/galp/src/engine/enums.cu new file mode 100644 index 00000000..1d2f7fac --- /dev/null +++ b/galp/src/engine/enums.cu @@ -0,0 +1,163 @@ +#include "engine/enums.cuh" +#include + +namespace enums { +DataType string_to_data_type(const std::string& str) { + static const std::unordered_map mapping = { + {"u32", DataType::U32}, + {"u64", DataType::U64}, + {"f32", DataType::F32}, + {"f64", DataType::F64}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown kernel type: " + str); +} + +Kernel string_to_kernel(const std::string& str) { + static const std::unordered_map mapping = { + {"decompress", Kernel::Decompress}, + {"query", Kernel::Query}, + {"query-multi-column", Kernel::QueryMultiColumn}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown kernel type: " + str); +} + +Unpacker string_to_unpacker(const std::string& str) { + static const std::unordered_map mapping = { + {"dummy", Unpacker::Dummy}, + {"old-fls", Unpacker::OldFls}, + {"switch-case", Unpacker::SwitchCase}, + {"stateless", Unpacker::Stateless}, + {"stateless-branchless", Unpacker::StatelessBranchless}, + {"stateful-cache", Unpacker::StatefulCache}, + {"stateful-local-1", Unpacker::StatefulLocal1}, + {"stateful-local-2", Unpacker::StatefulLocal2}, + {"stateful-local-4", Unpacker::StatefulLocal4}, + {"stateful-shared-1", Unpacker::StatefulShared1}, + {"stateful-shared-2", Unpacker::StatefulShared2}, + {"stateful-shared-4", Unpacker::StatefulShared4}, + {"stateful-register-1", Unpacker::StatefulRegister1}, + {"stateful-register-2", Unpacker::StatefulRegister2}, + {"stateful-register-4", Unpacker::StatefulRegister4}, + {"stateful-register-branchless-1", Unpacker::StatefulRegisterBranchless1}, + {"stateful-register-branchless-2", Unpacker::StatefulRegisterBranchless2}, + {"stateful-register-branchless-4", Unpacker::StatefulRegisterBranchless4}, + {"stateful-branchless", Unpacker::StatefulBranchless}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown unpacker type: " + str); +} + +Patcher string_to_patcher(const std::string& str) { + static const std::unordered_map mapping = { + {"none", Patcher::None}, + {"dummy", Patcher::Dummy}, + {"stateless", Patcher::Stateless}, + {"stateful", Patcher::Stateful}, + {"naive", Patcher::Naive}, + {"naive-branchless", Patcher::NaiveBranchless}, + {"prefetch-position", Patcher::PrefetchPosition}, + {"prefetch-all", Patcher::PrefetchAll}, + {"prefetch-all-branchless", Patcher::PrefetchAllBranchless}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown patcher type: " + str); +} +} // namespace enums + +namespace enums_nvcomp { +ComparisonType string_to_comparison_type(const std::string& str) { + static const std::unordered_map mapping = { + {"decompression", ComparisonType::DECOMPRESSION}, + {"decompression_query", ComparisonType::DECOMPRESSION_QUERY}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown comparison type: " + str); +} + +std::string comparison_type_to_string(const ComparisonType type) { + switch (type) { + case ComparisonType::DECOMPRESSION: + return "decompression"; + case ComparisonType::DECOMPRESSION_QUERY: + return "decompression_query"; + default: + throw std::invalid_argument("Could not parse comparison type"); + } +} + +CompressionType string_to_compression_type(const std::string& str) { + static const std::unordered_map mapping = { + {"Thrust", CompressionType::THRUST}, + {"ALP", CompressionType::ALP}, + {"GALP", CompressionType::GALP}, + {"Bitcomp", CompressionType::BITCOMP}, + {"BitcompSparse", CompressionType::BITCOMP_SPARSE}, + {"LZ4", CompressionType::LZ4}, + {"zstd", CompressionType::ZSTD}, + {"Deflate", CompressionType::DEFLATE}, + {"GDeflate", CompressionType::GDEFLATE}, + {"Snappy", CompressionType::SNAPPY}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown compression type: " + str); +} + +std::string compression_type_to_string(const CompressionType type) { + switch (type) { + case CompressionType::THRUST: + return "Thrust"; + case CompressionType::ALP: + return "ALP"; + case CompressionType::GALP: + return "GALP"; + case CompressionType::BITCOMP: + return "Bitcomp"; + case CompressionType::BITCOMP_SPARSE: + return "BitcompSparse"; + case CompressionType::LZ4: + return "LZ4"; + case CompressionType::ZSTD: + return "zstd"; + case CompressionType::DEFLATE: + return "Deflate"; + case CompressionType::GDEFLATE: + return "GDeflate"; + case CompressionType::SNAPPY: + return "Snappy"; + default: + throw std::invalid_argument("Could not parse decompresor"); + } +} +} // namespace enums_nvcomp diff --git a/galp/src/include/alp/alp-bindings.cuh b/galp/src/include/alp/alp-bindings.cuh new file mode 100644 index 00000000..af0e986a --- /dev/null +++ b/galp/src/include/alp/alp-bindings.cuh @@ -0,0 +1,37 @@ +#ifndef ALP_BINDINGS_CUH +#define ALP_BINDINGS_CUH + +#include "alp.hpp" +#include "flsgpu/flsgpu-api.cuh" +#include +#include +#include +#include + +namespace alp { + +class EncodingException : public std::exception { +public: + using std::exception::what; + const char* what() { + return "Could not encode data with desired encoding."; + } +}; + +// Test if data can be decoded in specified type +template +bool is_compressable(const T* input_array, const size_t n_elements); + +template +flsgpu::host::ALPColumn +encode(const T* input_array, const size_t n_elements, const bool print_compression_info = false); + +template +T* decode(const flsgpu::host::ALPColumn column, T* output_array); + +template +T* decode(const flsgpu::host::ALPExtendedColumn column, T* output_array); + +} // namespace alp + +#endif // ALP_BINDINGS_CUH diff --git a/galp/src/include/engine/data.cuh b/galp/src/include/engine/data.cuh new file mode 100644 index 00000000..84668b4a --- /dev/null +++ b/galp/src/include/engine/data.cuh @@ -0,0 +1,545 @@ +#ifndef DATA_CUH +#define DATA_CUH + +#include "alp.hpp" +#include "alp/alp-bindings.cuh" +#include "fls/unffor.hpp" +#include "fls_gen/pack/pack.hpp" +#include "fls_gen/unpack/unpack.hpp" +#include "flsgpu/flsgpu-api.cuh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace data { + +namespace primitives { + +template +std::function get_random_number_generator(const T min, const T max) { + std::random_device random_device; + std::default_random_engine random_engine(random_device()); + std::uniform_int_distribution uniform_dist(min, max); + + return std::bind(uniform_dist, random_engine); +} + +template +T* fill_array_with_constant(T* array, const size_t n_values, const T value) { + for (size_t i {0}; i < n_values; ++i) { + array[i] = value; + } + + return array; +} + +template +T* fill_array_with_random_bytes(T* array, const size_t n_values, const unsigned repeat = 1) { + // WARNING Does not check if n_values % REPEAT == 0 + using UINT_T = typename utils::same_width_uint::type; + UINT_T* out = reinterpret_cast(array); + auto generator = + get_random_number_generator(std::numeric_limits::min(), std::numeric_limits::max()); + + for (size_t i {0}; i < n_values; i += repeat) { + UINT_T value = generator(); + + for (size_t r {0}; r < repeat; ++r) { + out[i + r] = value; + } + } + + return array; +} + +template +T* fill_array_with_sequence( + T* array, const size_t n_values, const T start, const T step = 1, const unsigned repeat = 1) { + // WARNING Does not check if n_values % REPEAT == 0 + using UINT_T = typename utils::same_width_uint::type; + UINT_T* out = reinterpret_cast(array); + T value = start; + + for (size_t i {0}; i < n_values; i += repeat) { + for (size_t r {0}; r < repeat; ++r) { + out[i + r] = value; + } + + value += step; + } + + return array; +} + +template +T* fill_array_with_random_data(T* array, + const size_t n_values, + const unsigned repeat = 1, + const T min = std::numeric_limits::min(), + const T max = std::numeric_limits::max()) { + // WARNING Does not check if n_values % REPEAT == 0 + using UINT_T = typename utils::same_width_uint::type; + UINT_T* out = reinterpret_cast(array); + auto generator = get_random_number_generator(min, max); + + for (size_t i {0}; i < n_values; i += repeat) { + UINT_T value = generator(); + + for (size_t r {0}; r < repeat; ++r) { + out[i + r] = value; + } + } + + return array; +} + +template +T_sum sum_array(const T* array, const size_t n_values) { + T_sum sum = 0; + + for (size_t i {0}; i < n_values; ++i) { + sum += static_cast(array[i]); + } + + return sum; +} + +template +T_out* prefix_sum_array(const T_in* in, T_out* out, const size_t n_values) { + T_out sum = 0; + T_out old_sum = 0; + + for (size_t i {0}; i < n_values; ++i) { + old_sum = sum; + sum += in[i]; + out[i] = old_sum; + } + + return out; +} + +template +T* map(LambdaT lambda, T* array, const size_t n_values) { + for (size_t i {0}; i < n_values; ++i) { + array[i] = lambda(array[i]); + } + + return array; +} + +template +std::tuple make_column_magic(T* data, const size_t n_values) { +} + +template +T* generate_positions(T* positions, const T* counts, const size_t n_vecs) { + // INFO Not really a primitive ... + T* indices = new T[consts::VALUES_PER_VECTOR]; + for (size_t i {0}; i < consts::VALUES_PER_VECTOR; ++i) { + indices[i] = i; + } + + // Shuffle them and copy to the positions array + std::random_device random_device; + auto rng = std::default_random_engine(random_device()); + T* c_positions = positions; + for (size_t vi {0}; vi < n_vecs; ++vi) { + std::shuffle(indices, indices + consts::VALUES_PER_VECTOR, rng); + std::memcpy(c_positions, indices, sizeof(T) * counts[vi]); + std::sort(c_positions, c_positions + counts[vi]); + c_positions += counts[vi]; + } + delete[] indices; + + return positions; +} + +} // namespace primitives + +template +struct ValueRange { + const T min; + const T max; + + ValueRange() + : min(std::numeric_limits::min()) + , max(std::numeric_limits::max()) { + } + ValueRange(const T a_value) + : min(a_value) + , max(a_value) { + } + ValueRange(const T a_start, const T a_end) + : min(a_start) + , max(std::max(a_start, a_end)) { + } + + ValueRange operator=(const ValueRange& other) { + return ValueRange(min, max); + } +}; + +namespace arrays { + +template +std::pair read_file_as(const std::string path, const size_t input_count) { + // Open file + std::ifstream inputFile(path, std::ios::binary | std::ios::ate); + if (!inputFile) { + throw std::invalid_argument("Could not open the specified file."); + } + // Get file size + const std::streamsize file_size = inputFile.tellg(); + inputFile.seekg(0, std::ios::beg); + + // Check file size to contain right type of data + bool file_size_is_multiple_of_T_size = static_cast(file_size) % static_cast(sizeof(T)) != 0; + if (file_size_is_multiple_of_T_size) { + throw std::invalid_argument("File size is incorrect, it is not a multiple of the type's size."); + } + + const size_t values_in_file = static_cast(file_size) / sizeof(T); + size_t count = input_count == 0 ? values_in_file : input_count; + count = count - (count % consts::VALUES_PER_VECTOR); + auto column = new T[count]; + + // Read either the file size, or the total number of values needed, + // whichever is smaller + const std::streamsize read_size = std::min(file_size, static_cast((count * sizeof(T)))); + if (!inputFile.read(reinterpret_cast(column), read_size)) { + throw std::invalid_argument("Failed to read file into column"); + } + + inputFile.close(); + + // Copy paste the values in file until the column is filled + if (values_in_file < count) { + size_t n_filled_values = values_in_file; + size_t n_empty_values_column = count - n_filled_values; + while (n_empty_values_column != 0) { + size_t n_values_to_copy = std::min(n_empty_values_column, values_in_file); + std::memcpy(column + n_filled_values, column, n_values_to_copy * sizeof(T)); + n_filled_values += n_values_to_copy; + n_empty_values_column -= n_values_to_copy; + } + } + + return std::make_pair(column, count); +} + +template +T* generate_index_array(const size_t n_values, const vbw_t value_bit_width) { + T mask = utils::h_set_first_n_bits(value_bit_width); + T* array = new T[n_values]; + + for (size_t i {0}; i < n_values; i++) { + array[i] = i & mask; + } + + return array; +} + +template +T* generate_random_array(const size_t n_values, const vbw_t value_bit_width) { + T max_value = utils::h_set_first_n_bits(value_bit_width); + return primitives::fill_array_with_random_data(new T[n_values], n_values, 1, T {0}, max_value); +} + +} // namespace arrays + +namespace bindings { + +template +flsgpu::host::BPColumn compress(const T* array, const size_t n_values, const vbw_t value_bit_width) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + size_t compressed_vector_size = utils::get_compressed_vector_size(value_bit_width); + size_t n_packed_values = n_vecs * compressed_vector_size; + + T* packed_array = new T[n_packed_values]; + T* c_packed_array = packed_array; + + for (size_t vi {0}; vi < n_vecs; ++vi) { + generated::pack::fallback::scalar::pack(array, c_packed_array, value_bit_width); + array += consts::VALUES_PER_VECTOR; + c_packed_array += compressed_vector_size; + } + + return flsgpu::host::BPColumn { + n_values, + n_packed_values, + packed_array, + primitives::fill_array_with_constant(new vbw_t[n_vecs], n_vecs, value_bit_width), + primitives::fill_array_with_sequence(new size_t[n_vecs], n_vecs, 0, compressed_vector_size), + }; +} + +template +T* decompress(const flsgpu::host::BPColumn column) { + T* out_array = new T[column.get_n_values()]; + T* c_out_array = out_array; + T* c_packed_array = column.packed_array; + + for (size_t vi {0}; vi < column.get_n_vecs(); ++vi) { + c_packed_array = column.packed_array + column.vector_offsets[vi]; + generated::unpack::fallback::scalar::unpack(c_packed_array, c_out_array, column.bit_widths[vi]); + c_out_array += consts::VALUES_PER_VECTOR; + } + + return out_array; +} + +template +T* decompress(const flsgpu::host::FFORColumn column) { + T* out_array = new T[column.get_n_values()]; + T* c_out_array = out_array; + T* c_packed_array = column.bp.packed_array + column.bp.vector_offsets[0]; + + for (size_t vi {0}; vi < column.get_n_vecs(); ++vi) { + c_packed_array = column.bp.packed_array + column.bp.vector_offsets[vi]; + fastlanes::generated::unffor::fallback::scalar::unffor( + c_packed_array, c_out_array, column.bp.bit_widths[vi], &column.bases[vi]); + c_out_array += consts::VALUES_PER_VECTOR; + } + + return out_array; +} + +template +T* decompress(const flsgpu::host::ALPColumn column) { + return alp::decode(column, new T[column.get_n_values()]); +} + +template +T* decompress(const flsgpu::host::ALPExtendedColumn column) { + return alp::decode(column, new T[column.get_n_values()]); +} + +} // namespace bindings + +namespace columns { + +template +flsgpu::host::BPColumn generate_index_bp_column(const size_t n_values, const vbw_t value_bit_width) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + size_t n_packed_values = n_vecs * utils::get_compressed_vector_size(value_bit_width); + T* array = arrays::generate_index_array(n_values, value_bit_width); + + auto column = bindings::compress(array, n_values, value_bit_width); + delete[] array; + return column; +} + +template +flsgpu::host::FFORColumn generate_index_ffor_column(const size_t n_values, const vbw_t value_bit_width) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + return flsgpu::host::FFORColumn { + generate_index_bp_column(n_values, value_bit_width), + primitives::fill_array_with_random_data(new T[n_vecs], n_vecs, 1, 0, 64), + }; +} + +template +flsgpu::host::BPColumn +generate_random_bp_column(const size_t n_values, const ValueRange value_bit_width, const int32_t repeat = 1) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + vbw_t* bit_widths = primitives::fill_array_with_random_data( + new vbw_t[n_vecs], n_vecs, repeat, value_bit_width.min, value_bit_width.max); + size_t n_packed_values = primitives::sum_array(bit_widths, n_vecs) * utils::get_n_lanes(); + + return flsgpu::host::BPColumn { + n_values, + n_packed_values, + primitives::fill_array_with_random_bytes(new T[n_packed_values], n_packed_values), + bit_widths, + primitives::map([](const size_t value) { return value * utils::get_n_lanes(); }, + primitives::prefix_sum_array(bit_widths, new size_t[n_vecs], n_vecs), + n_vecs), + }; +} + +template +flsgpu::host::FFORColumn generate_random_ffor_column(const size_t n_values, + const ValueRange value_bit_width, + const ValueRange bases, + const int32_t repeat = 1) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + return flsgpu::host::FFORColumn { + generate_random_bp_column(n_values, value_bit_width, repeat), + primitives::fill_array_with_random_data(new T[n_vecs], n_vecs, 1, bases.min, bases.max), + }; +} + +template +std::tuple> +generate_binary_bp_column(const size_t n_values, const ValueRange value_bit_width, const int32_t repeat = 1) { + T* data = primitives::fill_array_with_constant(new T[n_values], n_values, 0); + + auto generate_presence = primitives::get_random_number_generator(0, 100); + bool contains_magic_value = false; // generate_presence() < 50; + if (contains_magic_value) { + auto index_generator = primitives::get_random_number_generator(0, n_values - 1); + data[index_generator()] = consts::as::MAGIC_NUMBER; + } + + auto column = bindings::compress(data, n_values, value_bit_width.max); + delete data; + + return std::make_tuple(contains_magic_value, column); +} + +template +std::tuple> +generate_binary_ffor_column(const size_t n_values, const ValueRange value_bit_width, const int32_t repeat = 1) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + auto [contains_magic_value, bp_column] = generate_binary_bp_column(n_values, value_bit_width, repeat); + T base = 0; + + if (value_bit_width.max == 0) { + base = T {contains_magic_value}; + } + + return std::make_tuple(contains_magic_value, + flsgpu::host::FFORColumn { + bp_column, + primitives::fill_array_with_constant(new T[n_vecs], n_vecs, base), + }); +} + +template +flsgpu::host::ALPColumn generate_alp_column(const size_t n_values, + const ValueRange bit_width_range, + const ValueRange exceptions_per_vec, + const unsigned repeat = 1) { + static_assert(std::is_floating_point::value, "T should be a floating point type."); + using UINT_T = typename utils::same_width_uint::type; + + const size_t n_vecs = utils::get_n_vecs_from_size(n_values); + auto column = flsgpu::host::ALPColumn(); + column.ffor = generate_random_ffor_column(n_values, bit_width_range, ValueRange(2, 20), repeat); + + // Note we halve the frac and fact because otherwise you + // are more likely to have integer overflow in the decoding for some + // combinations of compression parameters + column.factor_indices = primitives::fill_array_with_random_data( + new uint8_t[n_vecs], n_vecs, 1, 0, static_cast(consts::as::FACT_ARR_COUNT / 2)); + column.fraction_indices = primitives::fill_array_with_random_data( + new uint8_t[n_vecs], n_vecs, 1, 0, static_cast(consts::as::FRAC_ARR_COUNT / 2)); + + column.counts = primitives::fill_array_with_random_data( + new uint16_t[n_vecs], n_vecs, 1, exceptions_per_vec.min, exceptions_per_vec.max); + + column.n_exceptions = primitives::sum_array(column.counts, n_vecs); + column.exceptions_offsets = primitives::prefix_sum_array(column.counts, new size_t[n_vecs], n_vecs); + column.exceptions = primitives::fill_array_with_random_bytes(new T[column.n_exceptions], column.n_exceptions); + column.positions = + primitives::generate_positions(new uint16_t[column.n_exceptions], column.counts, n_vecs); + + // Not supported (yet) + column.compressed_size_bytes_alp = 0; + column.compressed_size_bytes_alp_extended = 0; + + return column; +} + +template +flsgpu::host::ALPColumn modify_alp_exception_count(flsgpu::host::ALPColumn column, + const ValueRange exceptions_per_vec) { + const size_t n_values = column.ffor.bp.n_values; + const size_t n_vecs = utils::get_n_vecs_from_size(n_values); + + delete[] column.counts; + delete[] column.exceptions_offsets; + delete[] column.exceptions; + delete[] column.positions; + + // Copied from generate_alp_column + column.counts = primitives::fill_array_with_random_data( + new uint16_t[n_vecs], n_vecs, 1, exceptions_per_vec.min, exceptions_per_vec.max); + column.n_exceptions = primitives::sum_array(column.counts, n_vecs); + column.exceptions_offsets = primitives::prefix_sum_array(column.counts, new size_t[n_vecs], n_vecs); + column.exceptions = primitives::fill_array_with_random_bytes(new T[column.n_exceptions], column.n_exceptions); + column.positions = + primitives::generate_positions(new uint16_t[column.n_exceptions], column.counts, n_vecs); + + return column; +} + +template +flsgpu::host::ALPColumn modify_alp_value_bit_width(flsgpu::host::ALPColumn column, + const ValueRange bit_width_range, + const unsigned repeat = 1) { + using UINT_T = typename utils::same_width_uint::type; + flsgpu::host::free_column(column.ffor); + column.ffor = generate_random_ffor_column( + column.ffor.bp.n_values, bit_width_range, repeat, ValueRange(2, 20)); + return column; +} + +template +void shuffle_bit_widths(flsgpu::host::BPColumn column) { + std::random_device random_device; + auto rng = std::default_random_engine(random_device()); + std::shuffle(column.bit_widths, column.bit_widths + column.get_n_vecs(), rng); + + column.vector_offsets = primitives::map( + [](const vbw_t value) { return value * utils::get_n_lanes(); }, + primitives::prefix_sum_array(column.bit_widths, column.vector_offsets, column.get_n_vecs()), + column.get_n_vecs()); +} + +template +void shuffle_bit_widths(flsgpu::host::FFORColumn column) { + shuffle_bit_widths(column.bp); +} + +template +void shuffle_bit_widths(flsgpu::host::ALPColumn column) { + shuffle_bit_widths(column.ffor.bp); +} + +template +void shuffle_bit_widths(flsgpu::host::ALPExtendedColumn column) { + shuffle_bit_widths(column.ffor.bp); +} + +template +std::tuple get_value_to_query(const ColumnT column) { + // Returns: + // The chance is 50% that the column contains the value. + // To find a suitable value: + // 50%) pick random value from column + // 50%) use constant, check whether it exists in the column + T* decompressed = bindings::decompress(column); + T value = consts::as::MAGIC_NUMBER; + bool column_does_not_contain_value = true; + + auto generate_presence = primitives::get_random_number_generator(0, 100); + bool ensure_column_contains_value = generate_presence() < 50; + if (ensure_column_contains_value) { + auto generate_index = primitives::get_random_number_generator(0, column.get_n_values() - 1); + value = decompressed[generate_index()]; + column_does_not_contain_value = false; + } else { + for (size_t i {0}; i < column.get_n_values(); ++i) { + column_does_not_contain_value &= value != decompressed[i]; + } + } + + delete decompressed; + return std::make_tuple(!column_does_not_contain_value, value); +} + +} // namespace columns + +} // namespace data + +#endif // DATA_CUH \ No newline at end of file diff --git a/galp/src/include/engine/device-utils.cuh b/galp/src/include/engine/device-utils.cuh new file mode 100644 index 00000000..18e59d1c --- /dev/null +++ b/galp/src/include/engine/device-utils.cuh @@ -0,0 +1,140 @@ +#include "flsgpu/flsgpu-api.cuh" +#include +#include + +#ifndef GPU_DEVICE_UTILS_CUH +#define GPU_DEVICE_UTILS_CUH + +template +struct SingleVectorPerWarpThreadblockMapping { + static constexpr unsigned N_WARPS_PER_BLOCK = std::max(utils::get_n_lanes() / consts::THREADS_PER_WARP, 2); + static constexpr unsigned N_THREADS_PER_BLOCK = N_WARPS_PER_BLOCK * consts::THREADS_PER_WARP; + static constexpr unsigned N_CONCURRENT_VECTORS_PER_BLOCK = + N_THREADS_PER_BLOCK / std::max(utils::get_n_lanes(), consts::THREADS_PER_WARP); + + const unsigned n_blocks; + + SingleVectorPerWarpThreadblockMapping(const size_t unpack_n_vecs, const size_t n_vecs) + : n_blocks(n_vecs / (unpack_n_vecs * N_CONCURRENT_VECTORS_PER_BLOCK)) { + } +}; +template +struct FillWarpThreadblockMapping { + static constexpr unsigned N_WARPS_PER_BLOCK = std::max(utils::get_n_lanes() / consts::THREADS_PER_WARP, 2); + static constexpr unsigned N_THREADS_PER_BLOCK = N_WARPS_PER_BLOCK * consts::THREADS_PER_WARP; + static constexpr unsigned N_CONCURRENT_VECTORS_PER_BLOCK = N_THREADS_PER_BLOCK / utils::get_n_lanes(); + + const unsigned n_blocks; + + FillWarpThreadblockMapping(const size_t unpack_n_vecs, const size_t n_vecs) + : n_blocks(n_vecs / (unpack_n_vecs * N_CONCURRENT_VECTORS_PER_BLOCK)) { + } +}; + +#ifdef SingleVectorMapping +template +using ThreadblockMapping = SingleVectorPerWarpThreadblockMapping; +#else +template +using ThreadblockMapping = FillWarpThreadblockMapping; +#endif + +struct VectorToWarpMappingBase { + virtual __device__ __forceinline__ lane_t get_lane() const; + virtual __device__ __forceinline__ vi_t get_vector_index() const; +}; + +template +struct SingleVectorPerWarpMapping : VectorToWarpMappingBase { + static constexpr uint32_t N_LANES = utils::get_n_lanes(); + static constexpr uint32_t N_VALUES_IN_LANE = utils::get_values_per_lane(); + + __device__ __forceinline__ lane_t get_lane() const override { + return threadIdx.x % N_LANES; + } + + __device__ __forceinline__ vi_t get_vector_index() const override { + // Concurrent vectors per block: how many vectors can be processed + // by the block simultaneously, assuming that each thread is 1 lane + + const int32_t concurrent_vectors_per_block = + blockDim.x / std::max(N_LANES, uint32_t {consts::THREADS_PER_WARP}); + const int32_t vectors_per_block = concurrent_vectors_per_block * UNPACK_N_VECTORS; + + const int32_t concurrent_vector_index = threadIdx.x / N_LANES; + const int32_t block_index = blockIdx.x; + + return vectors_per_block * block_index + concurrent_vector_index * UNPACK_N_VECTORS; + } +}; + +template +struct FillWarpMapping : VectorToWarpMappingBase { + static constexpr uint32_t N_LANES = utils::get_n_lanes(); + static constexpr uint32_t N_VALUES_IN_LANE = utils::get_values_per_lane(); + + __device__ __forceinline__ lane_t get_lane() const override { + return threadIdx.x % N_LANES; + } + + __device__ __forceinline__ vi_t get_vector_index() const override { + // Concurrent vectors per block: how many vectors can be processed + // by the block simultaneously, assuming that each thread is 1 lane + + const int32_t concurrent_vectors_per_block = blockDim.x / N_LANES; + const int32_t vectors_per_block = concurrent_vectors_per_block * UNPACK_N_VECTORS; + + const int32_t concurrent_vector_index = threadIdx.x / N_LANES; + const int32_t block_index = blockIdx.x; + + return vectors_per_block * block_index + concurrent_vector_index * UNPACK_N_VECTORS; + } +}; + +#ifdef SingleVectorMapping +template +using VectorToWarpMapping = OneVectorPerWarpMapping; +#else +template +using VectorToWarpMapping = FillWarpMapping; +#endif + +template +__device__ __forceinline__ void write_registers_to_global(const lane_t lane, + const si_t index_offset, + const T* __restrict registers, + T* __restrict out) { + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + for (int w {0}; w < UNPACK_N_VALUES; ++w) { + out[lane + (index_offset + w) * N_LANES + v * consts::VALUES_PER_VECTOR] = + registers[w + v * UNPACK_N_VALUES]; + } + } +} + +template +struct MagicChecker { + const T magic_value; + bool no_magic_found = true; + + __device__ __forceinline__ MagicChecker(const T magic_value) + : magic_value(magic_value) { + } + + __device__ __forceinline__ void check(const T* __restrict registers) { +#pragma unroll + for (int i = 0; i < N_VALUES; ++i) { + no_magic_found &= registers[i] != magic_value; + } + } + + __device__ __forceinline__ void write_result(bool* __restrict out) { + // This is a branch, as we do not want to write 0s, only emit a write + // if we found a magic value + if (!no_magic_found) { + *out = true; + } + } +}; + +#endif // GPU_DEVICE_UTILS_CUH diff --git a/galp/src/include/engine/enums.cuh b/galp/src/include/engine/enums.cuh new file mode 100644 index 00000000..f573a62a --- /dev/null +++ b/galp/src/include/engine/enums.cuh @@ -0,0 +1,94 @@ +#ifndef ENUMS_CUH +#define ENUMS_CUH + +#include +#include + +namespace enums { +enum class DataType { + U32, + U64, + F32, + F64, +}; + +enum class Kernel { + Decompress, + Query, + QueryMultiColumn, +}; + +enum class Unpacker { + Dummy, + OldFls, + SwitchCase, + Stateless, + StatelessBranchless, + StatefulCache, + StatefulLocal1, + StatefulLocal2, + StatefulLocal4, + StatefulShared1, + StatefulShared2, + StatefulShared4, + StatefulRegister1, + StatefulRegister2, + StatefulRegister4, + StatefulRegisterBranchless1, + StatefulRegisterBranchless2, + StatefulRegisterBranchless4, + StatefulBranchless, +}; + +enum class Patcher { + None, + Dummy, + Stateless, + Stateful, + Naive, + NaiveBranchless, + PrefetchPosition, + PrefetchAll, + PrefetchAllBranchless, +}; + +enum class Print { + PrintNothing, + PrintDebug, + PrintDebugExit0, +}; + +DataType string_to_data_type(const std::string& str); +Kernel string_to_kernel(const std::string& str); +Unpacker string_to_unpacker(const std::string& str); +Patcher string_to_patcher(const std::string& str); + +} // namespace enums + +namespace enums_nvcomp { +enum ComparisonType { + DECOMPRESSION, + DECOMPRESSION_QUERY, +}; + +ComparisonType string_to_comparison_type(const std::string& str); +std::string comparison_type_to_string(const ComparisonType type); + +enum CompressionType { + THRUST, + ALP, + GALP, + BITCOMP, + BITCOMP_SPARSE, + LZ4, + ZSTD, + DEFLATE, + GDEFLATE, + SNAPPY, +}; + +CompressionType string_to_compression_type(const std::string& str); +std::string compression_type_to_string(const CompressionType type); +} // namespace enums_nvcomp + +#endif // ENUMS_CUH diff --git a/galp/src/include/engine/kernels.cuh b/galp/src/include/engine/kernels.cuh new file mode 100644 index 00000000..884ff0a0 --- /dev/null +++ b/galp/src/include/engine/kernels.cuh @@ -0,0 +1,152 @@ +#include "engine/device-utils.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include + +#ifndef FLS_GLOBAL_CUH +#define FLS_GLOBAL_CUH + +namespace kernels { + +namespace device { + +template +__global__ void decompress_column(const ColumnT column, T* out) { + constexpr uint32_t N_VALUES = UNPACK_N_VALUES * UNPACK_N_VECTORS; + const auto mapping = VectorToWarpMapping(); + const lane_t lane = mapping.get_lane(); + const int32_t vector_index = mapping.get_vector_index(); + + T registers[N_VALUES]; + out += vector_index * consts::VALUES_PER_VECTOR; + + auto iterator = DecompressorT(column, vector_index, lane); + + for (si_t i = 0; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + iterator.unpack_next_into(registers); + + write_registers_to_global(lane, i, registers, out); + } +} + +template +__global__ void query_column(const ColumnT column, bool* out, const T magic_value) { + constexpr uint32_t N_VALUES = UNPACK_N_VALUES * UNPACK_N_VECTORS; + const auto mapping = VectorToWarpMapping(); + const lane_t lane = mapping.get_lane(); + const int32_t vector_index = mapping.get_vector_index(); + + T registers[N_VALUES]; + auto checker = MagicChecker(magic_value); + + DecompressorT unpacker = DecompressorT(column, vector_index, lane); + + for (si_t i = 0; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + unpacker.unpack_next_into(registers); + checker.check(registers); + } + + checker.write_result(out); +} + +template +__global__ void compute_column(const ColumnT column, bool* __restrict out, const T runtime_zero) { + constexpr T RANDOM_VALUE = 3; + constexpr uint32_t N_VALUES = UNPACK_N_VALUES * UNPACK_N_VECTORS; + const auto mapping = VectorToWarpMapping(); + const lane_t lane = mapping.get_lane(); + const vi_t vector_index = mapping.get_vector_index(); + + T registers[N_VALUES]; + auto checker = MagicChecker(1); + DecompressorT decompressor = DecompressorT(column, vector_index, lane); + + for (si_t i = 0; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor.unpack_next_into(registers); + +#pragma unroll + for (int32_t j {0}; j < N_VALUES; ++j) { +#pragma unroll + for (int32_t k {0}; k < N_REPETITIONS; ++k) { + registers[j] *= RANDOM_VALUE; + registers[j] <<= RANDOM_VALUE; + registers[j] += RANDOM_VALUE; + registers[j] >>= RANDOM_VALUE; + registers[j] &= runtime_zero; + } + } + + checker.check(registers); + } + + checker.write_result(out); +} +} // namespace device + +namespace host { + +template +__host__ T* decompress_column(const ColumnT column, const uint32_t n_samples) { + size_t n_vecs = utils::get_n_vecs_from_size(column.n_values); + const ThreadblockMapping mapping(UNPACK_N_VECTORS, n_vecs); + GPUArray device_out(column.n_values); + + for (uint32_t i {0}; i < n_samples; ++i) { + device::decompress_column + <<>>(column, device_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + T* out = new T[column.n_values]; + device_out.copy_to_host(out); + return out; +} + +template +__host__ bool query_column(const ColumnT column, const T magic_value, const uint32_t n_samples) { + size_t n_vecs = utils::get_n_vecs_from_size(column.n_values); + const ThreadblockMapping mapping(UNPACK_N_VECTORS, n_vecs); + bool result = false; + GPUArray device_out(1, &result); + + for (uint32_t i {0}; i < n_samples; ++i) { + device::query_column + <<>>(column, device_out.get(), magic_value); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + device_out.copy_to_host(&result); + return result; +} + +template +__host__ bool compute_column(const ColumnT column, const uint32_t n_samples) { + size_t n_vecs = utils::get_n_vecs_from_size(column.n_values); + const ThreadblockMapping mapping(UNPACK_N_VECTORS, n_vecs); + GPUArray device_out(1); + bool result; + + for (uint32_t i {0}; i < n_samples; ++i) { + device::compute_column + <<>>(column, device_out.get(), 0); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + device_out.copy_to_host(&result); + return result; +} + +} // namespace host + +} // namespace kernels + +#endif // FLS_GLOBAL_CUH diff --git a/galp/src/include/engine/multi-column-host-kernels.cuh b/galp/src/include/engine/multi-column-host-kernels.cuh new file mode 100644 index 00000000..89f94d73 --- /dev/null +++ b/galp/src/include/engine/multi-column-host-kernels.cuh @@ -0,0 +1,141 @@ +#ifndef MULTI_COLUMN_HOST_KERNELS_CUH +#define MULTI_COLUMN_HOST_KERNELS_CUH + +#include "data.cuh" +#include "device-utils.cuh" +#include "generated-bindings/multi-column-device-kernels.cuh" + +namespace kernels { namespace host { + +template +__host__ bool query_multi_column(const ColumnT column, const T magic_value, const uint32_t n_samples) { + using DeviceColumnT = typename ColumnT::DeviceColumnT; + constexpr int32_t MAX_N_COLS = 10; + DeviceColumnT device_columns[MAX_N_COLS]; + + for (int32_t c {0}; c < MAX_N_COLS; ++c) { + // INFO: Bitwidths are shuffled to ensure that the vectors that + // are unpacked in the same loop do not have an identical bitwidth, + // as this might benefit branched unpackers + data::columns::shuffle_bit_widths(column); + device_columns[c] = column.copy_to_device(); + } + + bool result = false; + GPUArray d_out(1, &result); + const ThreadblockMapping mapping(UNPACK_N_VECS, column.get_n_vecs()); + + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>( + device_columns[0], device_columns[1], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>( + device_columns[0], device_columns[1], device_columns[2], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>( + device_columns[0], device_columns[1], device_columns[2], device_columns[3], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + device_columns[7], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + device_columns[7], + device_columns[8], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + device_columns[7], + device_columns[8], + device_columns[9], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + for (int32_t c {0}; c < MAX_N_COLS; ++c) { + flsgpu::host::free_column(device_columns[c]); + } + + d_out.copy_to_host(&result); + return result; +} +}} // namespace kernels::host +#endif // MULTI_COLUMN_HOST_KERNELS_CUH diff --git a/galp/src/include/engine/verification.cuh b/galp/src/include/engine/verification.cuh new file mode 100644 index 00000000..6f23cfde --- /dev/null +++ b/galp/src/include/engine/verification.cuh @@ -0,0 +1,118 @@ +#ifndef VERIFICATION_H +#define VERIFICATION_H + +#include "flsgpu/flsgpu-api.cuh" +#include +#include +#include + +namespace verification { + +constexpr size_t LOG_N_MISTAKES = 5; + +template +struct Difference { + size_t index; + T original; + T other; + + template ::value, bool> = true> + void log() { + fprintf(stderr, + "[%lu] correct: %lu, found: %lu\n", + static_cast(index), + static_cast(original), + static_cast(other)); + } + + template ::value, bool> = true> + void log() { + using UINT_T = typename utils::same_width_uint::type; + + UINT_T* original_c = reinterpret_cast(&original); + UINT_T* other_c = reinterpret_cast(&other); + + if (sizeof(U) == 8) { + fprintf(stderr, + "[%lu] correct: %f (%016lX), found: %f (%016lX)\n", + index, + static_cast(original), + static_cast(*original_c), + static_cast(other), + static_cast(*other_c)); + } else { + fprintf(stderr, + "[%lu] correct: %f (%08X), found: %f (%08X)\n", + index, + static_cast(original), + static_cast(*original_c), + static_cast(other), + static_cast(*other_c)); + } + } +}; + +template +struct ExecutionResult { + bool success; + std::vector> differences; +}; + +template +bool byte_compare(const T a, const T b) { + using UINT_T = typename utils::same_width_uint::type; + const UINT_T* a_c = reinterpret_cast(&a); + const UINT_T* b_c = reinterpret_cast(&b); + + return (*a_c) == (*b_c); +} + +template +ExecutionResult compare_data(const T* a, const T* b, const size_t size) { + auto differences = std::vector>(); + + for (size_t i {0}; i < size; ++i) { + if (!byte_compare(a[i], b[i])) { + differences.push_back(Difference {i, a[i], b[i]}); + + if (differences.size() > LOG_N_MISTAKES) { + break; + } + } + } + + return ExecutionResult {differences.size() == 0, differences}; +} + +template +int32_t process_results(std::vector> results, bool print_debug) { + + int32_t runs_failed = 0; + for (size_t i {0}; i < results.size(); i++) { + if (!results[i].success) { + ++runs_failed; + + if (print_debug) { + fprintf(stderr, "\n Run %lu failed.\n", i); + + for (auto difference : results[i].differences) { + difference.template log(); + } + } + } + } + + if (print_debug) { + if (runs_failed == 0) { + fprintf(stderr, "Compression successful.\n"); + } else { + fprintf(stderr, "\n[%d/%ld] Runs failed.\n", runs_failed, results.size()); + } + } + + return runs_failed; +} + +} // namespace verification + +#endif // VERIFICATION_H diff --git a/galp/src/include/flsgpu/alp.cuh b/galp/src/include/flsgpu/alp.cuh new file mode 100644 index 00000000..d8b0a45b --- /dev/null +++ b/galp/src/include/flsgpu/alp.cuh @@ -0,0 +1,461 @@ +#ifndef ALP_CUH +#define ALP_CUH + +#include "fls.cuh" +#include "structs.cuh" +#include +#include +#include + +namespace flsgpu { namespace device { + +template +struct ALPFunctor : FunctorBase { +private: + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + + UINT_T bases[UNPACK_N_VECTORS]; + INT_T factor[UNPACK_N_VECTORS]; + T frac10[UNPACK_N_VECTORS]; + +public: + __device__ __forceinline__ ALPFunctor(const UINT_T* a_bases, + const INT_T* factors, + const uint8_t* factor_indices, + const T* fractions, + const uint8_t* fraction_indices) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + bases[v] = a_bases[v]; + factor[v] = factors[factor_indices[v]]; + frac10[v] = fractions[fraction_indices[v]]; + } + } + + __device__ __forceinline__ T operator()(const UINT_T value, const vi_t vector_index) override { + return static_cast(static_cast((value + bases[vector_index]) * factor[vector_index])) * + frac10[vector_index]; + } +}; + +template +struct ALPExceptionPatcherBase { +public: + virtual void __device__ __forceinline__ patch_if_needed(T* out); +}; + +template +struct DummyALPExceptionPatcher : flsgpu::device::ALPExceptionPatcherBase { + +public: + void __device__ __forceinline__ patch_if_needed(T* out) override { + } + + __device__ __forceinline__ + DummyALPExceptionPatcher(const flsgpu::device::ALPColumn column, const vi_t vector_index, const lane_t lane) { + } +}; + +template +struct StatelessALPExceptionPatcher : ALPExceptionPatcherBase { + using INT_T = typename utils::same_width_int::type; + + si_t start_index = 0; + uint16_t exceptions_count[UNPACK_N_VECTORS]; + uint16_t* vec_exceptions_positions[UNPACK_N_VECTORS]; + T* vec_exceptions[UNPACK_N_VECTORS]; + const lane_t lane; + +public: + void __device__ __forceinline__ patch_if_needed(T* out) override { + constexpr auto N_LANES = utils::get_n_lanes(); + + const int first_pos = start_index * N_LANES + lane; + const int last_pos = first_pos + N_LANES * (UNPACK_N_VALUES - 1); + + start_index += UNPACK_N_VALUES; + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + for (int i {0}; i < exceptions_count[v]; i++) { + auto position = vec_exceptions_positions[v][i]; + auto exception = vec_exceptions[v][i]; + if (position >= first_pos) { + if (position <= last_pos && position % N_LANES == lane) { + out[(position - first_pos) / N_LANES + v * UNPACK_N_VALUES] = exception; + } + if (position + 1 > last_pos) { + break; + } + } + } + } + } + + __device__ __forceinline__ + StatelessALPExceptionPatcher(const ALPColumn column, const vi_t first_vector_index, const lane_t lane) + : lane(lane) { + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + auto vec_index = first_vector_index + v; + exceptions_count[v] = column.counts[vec_index]; + vec_exceptions_positions[v] = column.positions + column.exceptions_offsets[vec_index]; + vec_exceptions[v] = column.exceptions + column.exceptions_offsets[vec_index]; + } + } +}; + +template +struct StatefulALPExceptionPatcher : ALPExceptionPatcherBase { + using INT_T = typename utils::same_width_int::type; + + si_t start_index = 0; + uint16_t exceptions_count[UNPACK_N_VECTORS]; + uint16_t* vec_exceptions_positions[UNPACK_N_VECTORS]; + T* vec_exceptions[UNPACK_N_VECTORS]; + const lane_t lane; + int32_t exception_index[UNPACK_N_VECTORS] = {0}; + +public: + void __device__ __forceinline__ patch_if_needed(T* out) override { + constexpr auto N_LANES = utils::get_n_lanes(); + + const int first_pos = start_index * N_LANES + lane; + const int last_pos = first_pos + N_LANES * (UNPACK_N_VALUES - 1); + start_index += UNPACK_N_VALUES; + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + for (; exception_index[v] < exceptions_count[v]; exception_index[v]++) { + auto position = vec_exceptions_positions[v][exception_index[v]]; + auto exception = vec_exceptions[v][exception_index[v]]; + if (position >= first_pos) { + if (position <= last_pos && position % N_LANES == lane) { + out[(position - first_pos) / N_LANES + v * UNPACK_N_VALUES] = exception; + } + if (position + 1 > last_pos) { + break; + } + } + } + } + } + + __device__ __forceinline__ + StatefulALPExceptionPatcher(const ALPColumn column, const vi_t first_vector_index, const lane_t lane) + : lane(lane) { + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + auto vec_index = first_vector_index + v; + exceptions_count[v] = column.counts[vec_index]; + vec_exceptions_positions[v] = column.positions + column.exceptions_offsets[vec_index]; + vec_exceptions[v] = column.exceptions + column.exceptions_offsets[vec_index]; + } + } +}; + +template +struct NaiveALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + uint16_t current_position; + +public: + __device__ __forceinline__ + NaiveALPExceptionPatcher(const ALPExtendedColumn column, const vi_t vector_index, const lane_t lane) + : current_position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t current_vector_index = vector_index + v; + + const auto offset_count = column.offsets_counts[current_vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto offset = (offset_count & 0x3FF); + positions[v] = column.positions + column.exceptions_offsets[current_vector_index] + offset; + exceptions[v] = column.exceptions + column.exceptions_offsets[current_vector_index] + offset; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + if (count[v] > 0 && current_position == *(positions[v])) { + out[v * UNPACK_N_VALUES + w] = *(exceptions[v]); + ++(positions[v]); + ++(exceptions[v]); + --(count[v]); + } + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +constexpr ToT __device__ __forceinline__ reinterpret_as(FromT value) { + ToT* ptr = reinterpret_cast(&value); + return *ptr; +} + +template +constexpr void __device__ __forceinline__ +overwrite_if_true(T* __restrict buffer, const T* __restrict new_value, const bool condition) { + using UINT_T = typename utils::same_width_uint::type; + *buffer = reinterpret_as((reinterpret_as(*buffer) * (!condition)) | + (reinterpret_as(*new_value) * condition)); +} + +template +struct NaiveBranchlessALPExceptionPatcher : ALPExceptionPatcherBase { +private: + using UINT_T = typename utils::same_width_uint::type; + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + uint16_t next_position; + uint16_t current_position; + +public: + __device__ __forceinline__ + NaiveBranchlessALPExceptionPatcher(const ALPExtendedColumn column, const vi_t vector_index, const lane_t lane) + : current_position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t current_vector_index = vector_index + v; + + const auto offset_count = column.offsets_counts[current_vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[current_vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + // It is possible to easily prefetch *positions[v] here + bool comp = (count[v] > 0) && (current_position == (*positions[v])); + overwrite_if_true(&out[v * UNPACK_N_VALUES + w], exceptions[v], comp); + positions[v] += comp; + exceptions[v] += comp; + count[v] -= comp; + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +struct PrefetchPositionALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + uint16_t next_position[UNPACK_N_VECTORS]; + uint16_t position; + +public: + __device__ __forceinline__ PrefetchPositionALPExceptionPatcher(const ALPExtendedColumn column, + const vi_t first_vector_index, + const lane_t lane) + : position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t vector_index = first_vector_index + v; + const auto offset_count = column.offsets_counts[vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + + next_position[v] = *positions[v]; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + if (count[v] > 0 && position == next_position[v]) { + out[v * UNPACK_N_VALUES + w] = *exceptions[v]; + ++positions[v]; + ++exceptions[v]; + --count[v]; + next_position[v] = *positions[v]; + } + } + position += utils::get_n_lanes(); + } + } +}; + +template +struct PrefetchAllALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + + uint16_t index[UNPACK_N_VECTORS] = {0}; + uint16_t next_position[UNPACK_N_VECTORS]; + T next_exception[UNPACK_N_VECTORS]; + uint16_t current_position; + +public: + void __device__ __forceinline__ read_next_exception(vi_t v) { + if (index[v] < count[v]) { + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + ++positions[v]; + ++exceptions[v]; + ++index[v]; + } else { + next_position[v] = consts::VALUES_PER_VECTOR; + } + } + + __device__ __forceinline__ + PrefetchAllALPExceptionPatcher(const ALPExtendedColumn column, const vi_t first_vector_index, const lane_t lane) + : current_position(lane) { + // Parse the data from the column +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const auto vector_index = first_vector_index + v; + const auto offset_count = column.offsets_counts[vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + + // This might be avoided to avoid a branch + read_next_exception(v); + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + if (current_position == next_position[v]) { + out[v * UNPACK_N_VALUES + w] = next_exception[v]; + read_next_exception(v); + } + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +struct PrefetchAllBranchlessALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + + uint16_t index[UNPACK_N_VECTORS] = {0}; + uint16_t next_position[UNPACK_N_VECTORS]; + T next_exception[UNPACK_N_VECTORS]; + uint16_t current_position; + +public: + void __device__ __forceinline__ read_next_exception() { + } + + __device__ __forceinline__ PrefetchAllBranchlessALPExceptionPatcher(const ALPExtendedColumn column, + const vi_t first_vector_index, + const lane_t lane) + : current_position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t vector_index = first_vector_index + v; + const auto offset_count = column.offsets_counts[vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + + bool comparison = count[v] > 0; + next_position[v] += (!comparison) * consts::VALUES_PER_VECTOR; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { + // NOTES: It is probably possible to remove the next_position variable + // as well as the next_exception, if you use prefetching. This would make + // it easier to do multiple vectors. + +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + bool comparison = current_position == next_position[v]; + overwrite_if_true(out + v * UNPACK_N_VALUES + w, &next_exception[v], comparison); + + positions[v] += comparison; + exceptions[v] += comparison; + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + index[v] += comparison; + + comparison = index[v] < count[v]; + next_position[v] += (!comparison) * consts::VALUES_PER_VECTOR; + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +struct ALPDecompressor : DecompressorBase { + UnpackerT unpacker; + PatcherT patcher; + + __device__ __forceinline__ ALPDecompressor(const ColumnT column, const vi_t vector_index, const lane_t lane) + : unpacker(column.ffor.bp.packed_array + column.ffor.bp.vector_offsets[vector_index], + lane, + column.ffor.bp.bit_widths[vector_index], + ALPFunctor(column.ffor.bases + vector_index, + column.factors, + column.factor_indices + vector_index, + column.fractions, + column.fraction_indices + vector_index)) + , patcher(PatcherT(column, vector_index, lane)) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + patcher.patch_if_needed(out); + } +}; + +}} // namespace flsgpu::device +#endif // ALP_CUH diff --git a/galp/src/include/flsgpu/consts.cuh b/galp/src/include/flsgpu/consts.cuh new file mode 100644 index 00000000..ca0f70db --- /dev/null +++ b/galp/src/include/flsgpu/consts.cuh @@ -0,0 +1,33 @@ +#ifndef CONSTS_H +#define CONSTS_H + +#include + +namespace consts { + +constexpr int32_t REGISTER_WIDTH = 1024; +constexpr int32_t VALUES_PER_VECTOR = 1024; +constexpr int32_t THREADS_PER_WARP = 32; +constexpr unsigned MAX_UNPACK_N_VECS = 4; + +template +struct as { + static inline constexpr T MAGIC_NUMBER = 1; +}; + +template <> +struct as { + static inline constexpr float MAGIC_NUMBER = 0.3214f; + static inline constexpr int32_t FACT_ARR_COUNT = 10; + static inline constexpr int32_t FRAC_ARR_COUNT = 11; +}; + +template <> +struct as { + static inline constexpr double MAGIC_NUMBER = 0.3214; + static inline constexpr int32_t FACT_ARR_COUNT = 19; + static inline constexpr int32_t FRAC_ARR_COUNT = 21; +}; +} // namespace consts + +#endif // CONSTS_H diff --git a/galp/src/include/flsgpu/device-types.cuh b/galp/src/include/flsgpu/device-types.cuh new file mode 100644 index 00000000..ae8d203e --- /dev/null +++ b/galp/src/include/flsgpu/device-types.cuh @@ -0,0 +1,11 @@ +#ifndef GPU_DEVICE_TYPES_CUH +#define GPU_DEVICE_TYPES_CUH + +#include + +using vbw_t = uint8_t; +using lane_t = uint16_t; +using si_t = uint32_t; // si = start index (of value within vector) +using vi_t = uint32_t; // vi = Vector Index + +#endif // GPU_DEVICE_TYPES_CUH diff --git a/galp/src/include/flsgpu/fls-switch-case.cuh b/galp/src/include/flsgpu/fls-switch-case.cuh new file mode 100644 index 00000000..efe13320 --- /dev/null +++ b/galp/src/include/flsgpu/fls-switch-case.cuh @@ -0,0 +1,891 @@ +#ifndef FLS_SWITCH_CASE_CUH +#define FLS_SWITCH_CASE_CUH + +#include "fls.cuh" + +namespace flsgpu { namespace device { + +template +__device__ void +switch_leaf(const typename utils::same_width_uint::type* __restrict in, T* __restrict out, processor_T processor) { + using UINT_T = typename utils::same_width_uint::type; + constexpr uint8_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + constexpr uint32_t N_LANES = utils::get_n_lanes(); + constexpr uint16_t PRECEDING_BITS = (START_INDEX * VALUE_BIT_WIDTH); + constexpr uint16_t BUFFER_OFFSET = PRECEDING_BITS % LANE_BIT_WIDTH; + constexpr uint16_t N_INPUT_LINE = PRECEDING_BITS / LANE_BIT_WIDTH; + + LoaderT loader(in + N_INPUT_LINE * N_LANES, utils::get_compressed_vector_size(VALUE_BIT_WIDTH)); + + constexpr T VALUE_MASK = utils::h_set_first_n_bits(VALUE_BIT_WIDTH); + + Masker masker(BUFFER_OFFSET, VALUE_BIT_WIDTH); + + UINT_T values[UNPACK_N_VECTORS]; + + // INFO Yes these lines contain if statements without constexpr + // INFO No this is not a problem, as you can check with a profiler + // whether the compiler is able to emit the right SASS, and the SASS + // is correct, with no run time branches, only 5-8 instructions per case + +#pragma unroll + for (int i = 0; i < UNPACK_N_VALUES; ++i) { + if (masker.is_buffer_empty()) { + loader.next_line(); + masker.next_line(); + } + + UINT_T buffers[UNPACK_N_VECTORS]; + loader.load_next_into(buffers); + masker.mask_and_increment(values, buffers); + + if (masker.continues_on_next_line()) { + loader.next_line(); + masker.next_line(); + loader.load_next_into(buffers); + masker.mask_and_insert_remaining_value(values, buffers); + } + +#pragma unroll + for (int v = 0; v < UNPACK_N_VECTORS; ++v) { + *(out + i + v * UNPACK_N_VALUES) = processor(values[v], v); + } + } +} + +template +__device__ void switch_start_index(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + processor_T processor, + const si_t start_index) { + constexpr int N_VALUES_IN_LANE = utils::get_values_per_lane(); + + switch (start_index) { + case 0: + if constexpr (0 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 1: + if constexpr (1 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 2: + if constexpr (2 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 3: + if constexpr (3 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 4: + if constexpr (4 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 5: + if constexpr (5 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 6: + if constexpr (6 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 7: + if constexpr (7 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 8: + if constexpr (8 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 9: + if constexpr (9 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 10: + if constexpr (10 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 11: + if constexpr (11 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 12: + if constexpr (12 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 13: + if constexpr (13 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 14: + if constexpr (14 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 15: + if constexpr (15 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 16: + if constexpr (16 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 17: + if constexpr (17 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 18: + if constexpr (18 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 19: + if constexpr (19 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 20: + if constexpr (20 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 21: + if constexpr (21 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 22: + if constexpr (22 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 23: + if constexpr (23 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 24: + if constexpr (24 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 25: + if constexpr (25 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 26: + if constexpr (26 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 27: + if constexpr (27 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 28: + if constexpr (28 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 29: + if constexpr (29 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 30: + if constexpr (30 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 31: + if constexpr (31 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 32: + if constexpr (32 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 33: + if constexpr (33 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 34: + if constexpr (34 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 35: + if constexpr (35 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 36: + if constexpr (36 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 37: + if constexpr (37 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 38: + if constexpr (38 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 39: + if constexpr (39 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 40: + if constexpr (40 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 41: + if constexpr (41 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 42: + if constexpr (42 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 43: + if constexpr (43 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 44: + if constexpr (44 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 45: + if constexpr (45 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 46: + if constexpr (46 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 47: + if constexpr (47 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 48: + if constexpr (48 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 49: + if constexpr (49 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 50: + if constexpr (50 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 51: + if constexpr (51 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 52: + if constexpr (52 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 53: + if constexpr (53 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 54: + if constexpr (54 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 55: + if constexpr (55 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 56: + if constexpr (56 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 57: + if constexpr (57 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 58: + if constexpr (58 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 59: + if constexpr (59 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 60: + if constexpr (60 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 61: + if constexpr (61 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 62: + if constexpr (62 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 63: + if constexpr (63 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + } +} + +template +__device__ void switch_value_bit_width(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + processor_T processor, + const vbw_t value_bit_width, + const si_t start_index) { + constexpr int N_BITS = utils::sizeof_in_bits(); + + switch (value_bit_width) { + case 0: + if constexpr (0 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 1: + if constexpr (1 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 2: + if constexpr (2 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 3: + if constexpr (3 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 4: + if constexpr (4 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 5: + if constexpr (5 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 6: + if constexpr (6 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 7: + if constexpr (7 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 8: + if constexpr (8 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 9: + if constexpr (9 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 10: + if constexpr (10 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 11: + if constexpr (11 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 12: + if constexpr (12 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 13: + if constexpr (13 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 14: + if constexpr (14 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 15: + if constexpr (15 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 16: + if constexpr (16 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 17: + if constexpr (17 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 18: + if constexpr (18 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 19: + if constexpr (19 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 20: + if constexpr (20 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 21: + if constexpr (21 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 22: + if constexpr (22 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 23: + if constexpr (23 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 24: + if constexpr (24 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 25: + if constexpr (25 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 26: + if constexpr (26 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 27: + if constexpr (27 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 28: + if constexpr (28 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 29: + if constexpr (29 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 30: + if constexpr (30 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 31: + if constexpr (31 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 32: + if constexpr (32 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 33: + if constexpr (33 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 34: + if constexpr (34 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 35: + if constexpr (35 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 36: + if constexpr (36 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 37: + if constexpr (37 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 38: + if constexpr (38 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 39: + if constexpr (39 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 40: + if constexpr (40 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 41: + if constexpr (41 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 42: + if constexpr (42 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 43: + if constexpr (43 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 44: + if constexpr (44 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 45: + if constexpr (45 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 46: + if constexpr (46 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 47: + if constexpr (47 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 48: + if constexpr (48 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 49: + if constexpr (49 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 50: + if constexpr (50 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 51: + if constexpr (51 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 52: + if constexpr (52 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 53: + if constexpr (53 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 54: + if constexpr (54 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 55: + if constexpr (55 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 56: + if constexpr (56 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 57: + if constexpr (57 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 58: + if constexpr (58 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 59: + if constexpr (59 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 60: + if constexpr (60 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 61: + if constexpr (61 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 62: + if constexpr (62 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 63: + if constexpr (63 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 64: + if constexpr (64 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + } +} + +template +struct BitUnpackerSwitchCase : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* __restrict in; + const vbw_t value_bit_width; + OutputProcessor processor; + si_t start_index = 0; + + __device__ __forceinline__ BitUnpackerSwitchCase(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(in + lane) + , value_bit_width(value_bit_width) + , processor(processor) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + switch_value_bit_width>( + in, out, processor, value_bit_width, start_index); + start_index += UNPACK_N_VALUES; + } +}; + +}} // namespace flsgpu::device + +#endif // FLS_SWITCH_CASE_CUH diff --git a/galp/src/include/flsgpu/fls.cuh b/galp/src/include/flsgpu/fls.cuh new file mode 100644 index 00000000..0e74883b --- /dev/null +++ b/galp/src/include/flsgpu/fls.cuh @@ -0,0 +1,648 @@ +#ifndef FLS_CUH +#define FLS_CUH + +#include "device-types.cuh" +#include "old-fls.cuh" +#include "structs.cuh" +#include "utils.cuh" +#include +#include +#include +#include + +namespace flsgpu { namespace device { + +template +struct BPFunctor : FunctorBase { + using UINT_T = typename utils::same_width_uint::type; + __device__ __forceinline__ BPFunctor() {}; + __device__ __forceinline__ T operator()(const UINT_T value, [[maybe_unused]] const vi_t vector_index) override { + return value; + } +}; + +template +struct FFORFunctor : FunctorBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T bases[UNPACK_N_VECTORS]; + __device__ __forceinline__ FFORFunctor(const UINT_T* a_bases) { +#pragma unroll + for (int32_t v {0}; v < UNPACK_N_VECTORS; ++v) { + bases[v] = a_bases[v]; + } + }; + + __device__ __forceinline__ UINT_T operator()(const UINT_T value, const vi_t vector_index) override { + return value + bases[vector_index]; + } +}; + +template +struct BitUnpackerBase { + /* Constructor, but cannot be enforced + BitUnpackerBase( + const UINT_T *__restrict in, const lane_t lane, + const vbw_t value_bit_width, OutputProcessor processor) + */ + virtual __device__ __forceinline__ void unpack_next_into(T* __restrict out); +}; + +template +struct BitUnpackerDummy : flsgpu::device::BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* in; + OutputProcessor processor; + + __device__ __forceinline__ BitUnpackerDummy(const UINT_T* __restrict a_in, + const lane_t lane, + [[maybe_unused]] const vbw_t value_bit_width, + OutputProcessor processor) + : in(a_in + lane) + , processor(processor) {}; + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + constexpr int32_t N_LANES = utils::get_n_lanes(); + +#pragma unroll + for (int v = 0; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int j = 0; j < UNPACK_N_VALUES; ++j) { + out[v * UNPACK_N_VALUES + j] = processor(in[v * consts::VALUES_PER_VECTOR + j * N_LANES], v); + } + } + + in += UNPACK_N_VALUES * N_LANES; + } +}; + +template +struct BitUnpackerOldFls : flsgpu::device::BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* in; + const vbw_t value_bit_width; + OutputProcessor processor; + + __device__ __forceinline__ BitUnpackerOldFls(const UINT_T* __restrict a_in, + const lane_t lane, + const vbw_t a_value_bit_width, + OutputProcessor processor) + : in(a_in + lane) + , value_bit_width(a_value_bit_width) + , processor(processor) { + static_assert(UNPACK_N_VECTORS == 1, "Old FLS can only unpack 1 at a time"); + static_assert(UNPACK_N_VALUES == utils::get_values_per_lane(), "Old FLS can only unpack entire lanes"); + }; + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + UINT_T* u_out = reinterpret_cast(out); + oldfls::adjusted::unpack(in, u_out, value_bit_width); + + for (int32_t i {0}; i < UNPACK_N_VALUES; ++i) { + out[i] = processor(u_out[i], 0); + } + } +}; + +template +struct LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + + virtual __device__ __forceinline__ void load_next_into(UINT_T* out); + virtual __device__ __forceinline__ void next_line(); +}; + +template +struct CacheLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* in; + int32_t vector_offset; + + __device__ __forceinline__ CacheLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) {}; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = *(in + v * vector_offset); + } + } + + __device__ __forceinline__ void next_line() override { + in += utils::get_n_lanes(); + } +}; + +template +struct LocalMemoryLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T buffers[UNPACK_N_VECTORS * BUFFER_SIZE]; + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + + __device__ __forceinline__ LocalMemoryLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + buffer_index]; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { + ++buffer_index; + } + } +}; + +template +struct SharedMemoryLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + // No syncthreads are needed as threads only read and write their own section + // Shared memory is allocated per block, so this also depends on block config + // 4 divide by 32 bits/lanes, multiply by number of warps per block + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + UINT_T* buffers; + + __device__ __forceinline__ SharedMemoryLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + constexpr uint32_t N_LANES = utils::get_n_lanes(); + __shared__ UINT_T shared_ptr[N_LANES * BUFFER_SIZE * UNPACK_N_VECTORS * (sizeof(T) / 4 * 2)]; + buffers = shared_ptr + threadIdx.x * BUFFER_SIZE * UNPACK_N_VECTORS; + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + buffer_index]; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { + ++buffer_index; + } + } +}; + +template +struct RegisterLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T buffers[UNPACK_N_VECTORS * BUFFER_SIZE]; + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + + __device__ __forceinline__ RegisterLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + static_assert(BUFFER_SIZE <= 4, "Switch in RegisterLoader is not long enough for this buffer size."); + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { + + switch (buffer_index) { + case 0: { + if (0 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 0]; + } + } + } break; + case 1: { + if (1 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 1]; + } + } + } break; + case 2: { + if (2 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 2]; + } + } + } break; + case 3: { + if (3 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 3]; + } + } + } break; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { + ++buffer_index; + } + } +}; + +template +struct RegisterBranchlessLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T buffers[UNPACK_N_VECTORS * BUFFER_SIZE]; + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + + __device__ __forceinline__ RegisterBranchlessLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE]; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {1}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b - 1] = buffers[v * BUFFER_SIZE + b]; + } + } + ++buffer_index; + } + } +}; + +template +struct Masker { + using UINT_T = typename utils::same_width_uint::type; + const vbw_t value_bit_width; + const T value_mask; + uint16_t buffer_offset = 0; + + __device__ __forceinline__ Masker(const vbw_t value_bit_width) + : value_bit_width(value_bit_width) + , value_mask(utils::set_first_n_bits(value_bit_width)) {}; + + __device__ __forceinline__ Masker(const uint16_t buffer_offset, const vbw_t value_bit_width) + : buffer_offset(buffer_offset) + , value_bit_width(value_bit_width) + , value_mask(utils::set_first_n_bits(value_bit_width)) {}; + + __device__ __forceinline__ void mask_and_increment(T* values, const T* buffers) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + values[v] = (buffers[v] & (value_mask << buffer_offset)) >> buffer_offset; + } + buffer_offset += value_bit_width; + } + + __device__ __forceinline__ void next_line() { + buffer_offset -= utils::get_lane_bitwidth(); + } + __device__ __forceinline__ bool is_buffer_empty() const { + return buffer_offset == utils::get_lane_bitwidth(); + } + + __device__ __forceinline__ bool continues_on_next_line() const { + return buffer_offset > utils::get_lane_bitwidth(); + } + + __device__ __forceinline__ void mask_and_insert_remaining_value(T* values, const T* buffers) const { + T buffer_offset_mask = (T {1} << static_cast(buffer_offset)) - T {1}; + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + values[v] |= (buffers[v] & buffer_offset_mask) << (value_bit_width - buffer_offset); + } + } +}; + +template +__device__ void unpack_vector_stateless(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + const lane_t lane, + const vbw_t value_bit_width, + const si_t start_index, + processor_T processor, + int32_t vector_offset) { + using UINT_T = typename utils::same_width_uint::type; + constexpr uint8_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + constexpr uint32_t N_LANES = utils::get_n_lanes(); + uint16_t preceding_bits = (start_index * value_bit_width); + uint16_t buffer_offset = preceding_bits % LANE_BIT_WIDTH; + uint16_t n_input_line = preceding_bits / LANE_BIT_WIDTH; + + LoaderT loader(in + n_input_line * N_LANES + lane, vector_offset); + Masker masker(buffer_offset, value_bit_width); + + UINT_T values[UNPACK_N_VECTORS]; + +#pragma unroll + for (int i = 0; i < UNPACK_N_VALUES; ++i) { + if (masker.is_buffer_empty()) { + loader.next_line(); + masker.next_line(); + } + + UINT_T buffers[UNPACK_N_VECTORS]; + loader.load_next_into(buffers); + masker.mask_and_increment(values, buffers); + + if (masker.continues_on_next_line()) { + loader.next_line(); + masker.next_line(); + loader.load_next_into(buffers); + masker.mask_and_insert_remaining_value(values, buffers); + } + +#pragma unroll + for (int v = 0; v < UNPACK_N_VECTORS; ++v) { + *(out + i + v * UNPACK_N_VALUES) = processor(values[v], v); + } + } +} + +template +struct BitUnpackerStateless : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* __restrict in; + const lane_t lane; + const vbw_t value_bit_width; + OutputProcessor processor; + int32_t vector_offset; + + si_t start_index = 0; + + __device__ __forceinline__ BitUnpackerStateless(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(in) + , lane(lane) + , value_bit_width(value_bit_width) + , processor(processor) + , vector_offset(utils::get_compressed_vector_size(value_bit_width)) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + unpack_vector_stateless>( + in, out, lane, value_bit_width, start_index, processor, vector_offset); + start_index += UNPACK_N_VALUES; + } +}; + +template +__device__ void unpack_vector_stateless_branchless(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + const lane_t lane, + const vbw_t value_bit_width, + const si_t start_index, + processor_T processor, + const int32_t vector_offset) { + using UINT_T = typename utils::same_width_uint::type; + constexpr int32_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + constexpr int32_t N_LANES = utils::get_n_lanes(); + constexpr int32_t BIT_COUNT = utils::sizeof_in_bits(); + + int32_t preceding_bits_first = (start_index * value_bit_width); + int32_t n_input_line = preceding_bits_first / LANE_BIT_WIDTH; + int32_t offset_first = preceding_bits_first % LANE_BIT_WIDTH; + int32_t offset_second = BIT_COUNT - offset_first; + UINT_T value_mask = utils::set_first_n_bits(value_bit_width); + + UINT_T values[UNPACK_N_VECTORS] = {0}; + + in += n_input_line * N_LANES + lane; +#pragma unroll + for (int32_t v {0}; v < UNPACK_N_VECTORS; v++) { + const auto v_in = in + v * vector_offset; + values[v] |= (v_in[0] & (value_mask << offset_first)) >> offset_first; + values[v] |= (v_in[N_LANES] & (value_mask >> offset_second)) << offset_second; + out[v] = processor(values[v], v); + } +} + +template +struct BitUnpackerStatelessBranchless : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* __restrict in; + const lane_t lane; + const vbw_t value_bit_width; + OutputProcessor processor; + const int32_t vector_offset; + + si_t start_index = 0; + + __device__ __forceinline__ BitUnpackerStatelessBranchless(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(in) + , lane(lane) + , value_bit_width(value_bit_width) + , processor(processor) + , vector_offset(utils::get_compressed_vector_size(value_bit_width)) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { +#pragma unroll + for (int32_t i {0}; i < UNPACK_N_VALUES; i++) { + unpack_vector_stateless_branchless( + in, out + i, lane, value_bit_width, start_index + i, processor, vector_offset); + } + start_index += UNPACK_N_VALUES; + } +}; + +template +struct BitUnpackerStateful : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + LoaderT loader; + Masker masker; + OutputProcessor processor; + + __device__ __forceinline__ BitUnpackerStateful(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : loader(in + lane, utils::get_compressed_vector_size(value_bit_width)) + , masker(value_bit_width) + , processor(processor) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + UINT_T values[UNPACK_N_VECTORS]; + +#pragma unroll + for (int i = 0; i < UNPACK_N_VALUES; ++i) { + if (masker.is_buffer_empty()) { + loader.next_line(); + masker.next_line(); + } + + UINT_T buffers[UNPACK_N_VECTORS]; + loader.load_next_into(buffers); + masker.mask_and_increment(values, buffers); + + if (masker.continues_on_next_line()) { + loader.next_line(); + masker.next_line(); + loader.load_next_into(buffers); + masker.mask_and_insert_remaining_value(values, buffers); + } + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + *(out + i + v * UNPACK_N_VALUES) = processor(values[v], v); + } + } + } +}; + +template +struct BitUnpackerStatefulBranchless : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + OutputProcessor processor; + + const UINT_T* in; + const int32_t vector_offset; + const vbw_t value_bit_width; + + int32_t offset_first = 0; + UINT_T value_mask; + + __device__ __forceinline__ BitUnpackerStatefulBranchless(const UINT_T* __restrict a_in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(a_in + lane) + , value_bit_width(value_bit_width) + , value_mask(utils::set_first_n_bits(value_bit_width)) + , vector_offset(utils::get_compressed_vector_size(value_bit_width)) + , processor(processor) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + constexpr int32_t N_LANES = utils::get_n_lanes(); + constexpr int32_t BIT_COUNT = utils::sizeof_in_bits(); + constexpr int32_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + +#pragma unroll + for (int32_t i {0}; i < UNPACK_N_VALUES; i++) { + const auto offset_second = BIT_COUNT - offset_first; + +#pragma unroll + for (int32_t v {0}; v < UNPACK_N_VECTORS; v++) { + const auto v_in = in + v * vector_offset; + out[UNPACK_N_VALUES * v + i] = + processor(((v_in[0] >> offset_first) & value_mask) | + ((v_in[N_LANES] & (value_mask >> offset_second)) << offset_second), + v); + } + + in += (offset_second <= value_bit_width) * N_LANES; + offset_first = (offset_first + value_bit_width) % LANE_BIT_WIDTH; + } + } +}; + +template +struct BPDecompressor : DecompressorBase { + UnpackerT unpacker; + __device__ __forceinline__ BPDecompressor(const BPColumn column, const vi_t vector_index, const lane_t lane) + : unpacker(column.packed_array + column.vector_offsets[vector_index], + lane, + column.bit_widths[vector_index], + BPFunctor()) { + } + + void __device__ unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + } +}; + +template +struct FFORDecompressor : DecompressorBase { + UnpackerT unpacker; + __device__ __forceinline__ FFORDecompressor(const FFORColumn column, const vi_t vector_index, const lane_t lane) + : unpacker(column.bp.packed_array + column.bp.vector_offsets[vector_index], + lane, + column.bp.bit_widths[vector_index], + FFORFunctor(column.bases + vector_index)) { + } + + void __device__ unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + } +}; + +}} // namespace flsgpu::device + +#endif // FLS_CUH diff --git a/galp/src/include/flsgpu/flsgpu-api.cuh b/galp/src/include/flsgpu/flsgpu-api.cuh new file mode 100644 index 00000000..3926ee6d --- /dev/null +++ b/galp/src/include/flsgpu/flsgpu-api.cuh @@ -0,0 +1,4 @@ +#include "alp.cuh" +#include "fls-switch-case.cuh" +#include "fls.cuh" +#include "structs.cuh" diff --git a/galp/src/include/flsgpu/host-utils.cuh b/galp/src/include/flsgpu/host-utils.cuh new file mode 100644 index 00000000..159e99d5 --- /dev/null +++ b/galp/src/include/flsgpu/host-utils.cuh @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifndef GPU_UTILS_H +#define GPU_UTILS_H + +#define CUDA_SAFE_CALL(call) \ + do { \ + cudaError_t err = call; \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in file '%s' in line %i : %s.", __FILE__, __LINE__, cudaGetErrorString(err)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define CUDA_SAFE_CALL_TRACED(call) \ + do { \ + fprintf(stderr, "Start CUDA_CALL ['%s': line %i\n", __FILE__, __LINE__); \ + cudaError_t err = call; \ + fprintf(stderr, "End CUDA_CALL ['%s': line %i\n", __FILE__, __LINE__); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in file '%s' in line %i : %s.", __FILE__, __LINE__, cudaGetErrorString(err)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +template +void free_device_pointer(T*& device_ptr) { + if (device_ptr != nullptr) { + CUDA_SAFE_CALL(cudaFree(device_ptr)); + } + device_ptr = nullptr; +} + +template +class GPUArray { +private: + size_t allocation_size; + size_t memory_size; + T* device_ptr = nullptr; + + void allocate() { + CUDA_SAFE_CALL(cudaMalloc(reinterpret_cast(&device_ptr), allocation_size)); + } + +public: + GPUArray(const size_t count) { + memory_size = count * sizeof(T); + allocation_size = memory_size; + allocate(); + } + + GPUArray(const size_t count, const T* host_p) { + memory_size = count * sizeof(T); + allocation_size = memory_size; + allocate(); + CUDA_SAFE_CALL(cudaMemcpy(device_ptr, host_p, memory_size, cudaMemcpyHostToDevice)); + } + + GPUArray(const size_t count, const size_t buffer, const T* host_p) { + memory_size = count * sizeof(T); + allocation_size = memory_size + buffer * sizeof(T); + allocate(); + + CUDA_SAFE_CALL(cudaMemcpy(device_ptr, host_p, memory_size, cudaMemcpyHostToDevice)); + } + + // Copy constructor + GPUArray(const GPUArray&) = delete; + // Assignment operator deleted + GPUArray& operator=(const GPUArray&) = delete; + + // Move constructor + GPUArray(GPUArray&& other) noexcept + : device_ptr(other.device_ptr) { + other.device_ptr = nullptr; + } + + // Assignment operator + GPUArray& operator=(GPUArray&& other) noexcept { + if (this != &other) { + free_device_pointer(device_ptr); + device_ptr = other.device_ptr; + other.device_ptr = nullptr; + } + return *this; + } + + ~GPUArray() { + free_device_pointer(device_ptr); + } + + void copy_to_host(T* host_p) { + CUDA_SAFE_CALL(cudaMemcpy(host_p, device_ptr, memory_size, cudaMemcpyDeviceToHost)); + } + + T* get() { + return device_ptr; + } + + T* release() { + auto temp = device_ptr; + device_ptr = nullptr; + return temp; + } +}; + +#endif // GPU_UTILS_H diff --git a/galp/src/include/flsgpu/old-fls.cuh b/galp/src/include/flsgpu/old-fls.cuh new file mode 100644 index 00000000..4b0af3bb --- /dev/null +++ b/galp/src/include/flsgpu/old-fls.cuh @@ -0,0 +1,6936 @@ +#ifndef OLD_FLS_KERNELS_H +#define OLD_FLS_KERNELS_H + +#include +#include +#include + +namespace oldfls { + +namespace original { +__device__ __forceinline__ void unpack_0bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + // int i = threadIdx.x; // THREAD INDEX + + out[0] = base_0; + out[1] = base_0; + out[2] = base_0; + out[3] = base_0; + out[4] = base_0; + out[5] = base_0; + out[6] = base_0; + out[7] = base_0; + out[8] = base_0; + out[9] = base_0; + out[10] = base_0; + out[11] = base_0; + out[12] = base_0; + out[13] = base_0; + out[14] = base_0; + out[15] = base_0; + out[16] = base_0; + out[17] = base_0; + out[18] = base_0; + out[19] = base_0; + out[20] = base_0; + out[21] = base_0; + out[22] = base_0; + out[23] = base_0; + out[24] = base_0; + out[25] = base_0; + out[26] = base_0; + out[27] = base_0; + out[28] = base_0; + out[29] = base_0; + out[30] = base_0; + out[31] = base_0; +} +__device__ __forceinline__ void unpack_1bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 1) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_2bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_3bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 3) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_4bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_5bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 5) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_6bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_7bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 7) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_8bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_9bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 9) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_10bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_11bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 11) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_12bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_13bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 13) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_14bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_15bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 15) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_16bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[1] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[5] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[9] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[13] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[17] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[21] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[25] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[29] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_17bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 17) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; + out[18] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; + out[22] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; + out[26] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; + out[30] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_18bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[10] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[14] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[26] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[30] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_19bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 19) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; + out[6] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; + out[18] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; + out[26] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; + out[30] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_20bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[6] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[14] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[22] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[30] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_21bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 21) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; + out[10] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; + out[18] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; + out[22] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; + out[30] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_22bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[2] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[14] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[18] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[30] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_23bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 23) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; + out[2] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; + out[6] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; + out[26] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; + out[30] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_24bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[2] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[6] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[10] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[14] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[18] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[22] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[26] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[30] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_25bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 25) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; + out[2] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; + out[6] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; + out[12] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; + out[26] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; + out[30] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_26bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[2] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[4] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[14] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[18] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[20] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[30] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_27bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 27) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; + out[4] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; + out[10] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; + out[18] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; + out[22] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; + out[30] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_28bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[6] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[14] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[22] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[30] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_29bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 29) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; + out[6] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; + out[18] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; + out[20] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; + out[26] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; + out[30] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_30bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[10] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[12] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[14] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[26] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[28] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[30] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_31bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 31) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; + out[18] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; + out[22] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; + out[26] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; + out[28] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 960); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; + out[30] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_32bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + out[0] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + out[1] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + out[2] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + out[3] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + out[4] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + out[5] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + out[6] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + out[7] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + out[8] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + out[9] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + out[10] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + out[11] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + out[12] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + out[13] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + out[14] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + out[15] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 512); + out[16] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 544); + out[17] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + out[18] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 608); + out[19] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 640); + out[20] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + out[21] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 704); + out[22] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 736); + out[23] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 768); + out[24] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 800); + out[25] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 832); + out[26] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 864); + out[27] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 896); + out[28] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 928); + out[29] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 960); + out[30] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 992); + out[31] = register_0; +} +__device__ __forceinline__ void unpack(const uint32_t* __restrict a_in_p, uint32_t* __restrict a_out_p, uint8_t bw) { + switch (bw) { + case 0: + unpack_0bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 1: + unpack_1bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 2: + unpack_2bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 3: + unpack_3bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 4: + unpack_4bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 5: + unpack_5bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 6: + unpack_6bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 7: + unpack_7bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 8: + unpack_8bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 9: + unpack_9bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 10: + unpack_10bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 11: + unpack_11bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 12: + unpack_12bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 13: + unpack_13bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 14: + unpack_14bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 15: + unpack_15bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 16: + unpack_16bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 17: + unpack_17bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 18: + unpack_18bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 19: + unpack_19bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 20: + unpack_20bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 21: + unpack_21bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 22: + unpack_22bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 23: + unpack_23bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 24: + unpack_24bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 25: + unpack_25bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 26: + unpack_26bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 27: + unpack_27bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 28: + unpack_28bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 29: + unpack_29bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 30: + unpack_30bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 31: + unpack_31bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 32: + unpack_32bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + } +} + +} // namespace original + +namespace adjusted { + +__device__ __forceinline__ void unpack_0bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + // int i = 0; // THREAD INDEX + + out[0] = base_0; + out[1] = base_0; + out[2] = base_0; + out[3] = base_0; + out[4] = base_0; + out[5] = base_0; + out[6] = base_0; + out[7] = base_0; + out[8] = base_0; + out[9] = base_0; + out[10] = base_0; + out[11] = base_0; + out[12] = base_0; + out[13] = base_0; + out[14] = base_0; + out[15] = base_0; + out[16] = base_0; + out[17] = base_0; + out[18] = base_0; + out[19] = base_0; + out[20] = base_0; + out[21] = base_0; + out[22] = base_0; + out[23] = base_0; + out[24] = base_0; + out[25] = base_0; + out[26] = base_0; + out[27] = base_0; + out[28] = base_0; + out[29] = base_0; + out[30] = base_0; + out[31] = base_0; +} +__device__ __forceinline__ void unpack_1bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 1) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_2bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_3bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 3) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_4bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_5bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 5) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_6bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_7bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 7) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_8bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_9bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 9) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_10bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_11bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 11) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_12bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_13bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 13) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_14bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_15bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 15) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_16bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[1] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[5] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[9] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[13] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[17] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[21] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[25] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[29] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_17bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 17) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; + out[18] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; + out[22] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; + out[26] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; + out[30] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_18bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[10] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[14] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[26] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[30] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_19bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 19) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; + out[6] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; + out[18] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; + out[26] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; + out[30] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_20bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[6] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[14] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[22] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[30] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_21bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 21) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; + out[10] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; + out[18] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; + out[22] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; + out[30] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_22bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[2] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[14] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[18] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[30] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_23bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 23) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; + out[2] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; + out[6] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; + out[26] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; + out[30] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_24bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[2] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[6] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[10] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[14] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[18] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[22] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[26] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[30] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_25bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 25) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; + out[2] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; + out[6] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; + out[12] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; + out[26] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; + out[30] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_26bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[2] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[4] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[14] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[18] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[20] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[30] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_27bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 27) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; + out[4] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; + out[10] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; + out[18] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; + out[22] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; + out[30] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_28bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[6] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[14] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[22] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[30] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_29bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 29) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; + out[6] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; + out[18] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; + out[20] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; + out[26] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; + out[30] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_30bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[10] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[12] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[14] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[26] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[28] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[30] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_31bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 31) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; + out[18] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; + out[22] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; + out[26] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; + out[28] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 960); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; + out[30] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_32bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + out[0] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + out[1] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + out[2] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + out[3] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + out[4] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + out[5] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + out[6] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + out[7] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + out[8] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + out[9] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + out[10] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + out[11] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + out[12] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + out[13] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + out[14] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + out[15] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 512); + out[16] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 544); + out[17] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + out[18] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 608); + out[19] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 640); + out[20] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + out[21] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 704); + out[22] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 736); + out[23] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 768); + out[24] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 800); + out[25] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 832); + out[26] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 864); + out[27] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 896); + out[28] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 928); + out[29] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 960); + out[30] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 992); + out[31] = register_0; +} +__device__ __forceinline__ void unpack(const uint32_t* __restrict a_in_p, uint32_t* __restrict a_out_p, uint8_t bw) { + switch (bw) { + case 0: + unpack_0bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 1: + unpack_1bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 2: + unpack_2bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 3: + unpack_3bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 4: + unpack_4bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 5: + unpack_5bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 6: + unpack_6bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 7: + unpack_7bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 8: + unpack_8bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 9: + unpack_9bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 10: + unpack_10bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 11: + unpack_11bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 12: + unpack_12bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 13: + unpack_13bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 14: + unpack_14bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 15: + unpack_15bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 16: + unpack_16bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 17: + unpack_17bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 18: + unpack_18bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 19: + unpack_19bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 20: + unpack_20bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 21: + unpack_21bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 22: + unpack_22bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 23: + unpack_23bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 24: + unpack_24bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 25: + unpack_25bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 26: + unpack_26bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 27: + unpack_27bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 28: + unpack_28bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 29: + unpack_29bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 30: + unpack_30bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 31: + unpack_31bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 32: + unpack_32bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + } +} + +} // namespace adjusted +} // namespace oldfls + +#endif // OLD_FLS_KERNELS_H diff --git a/galp/src/include/flsgpu/structs.cuh b/galp/src/include/flsgpu/structs.cuh new file mode 100644 index 00000000..bb40f290 --- /dev/null +++ b/galp/src/include/flsgpu/structs.cuh @@ -0,0 +1,411 @@ + +#ifndef STRUCTS_CUH +#define STRUCTS_CUH + +#include "alp.hpp" +#include "device-types.cuh" +#include "host-utils.cuh" +#include "utils.cuh" +#include +#include +#include + +namespace flsgpu { + +namespace device { + +template +struct FunctorBase { + using UINT_T = typename utils::same_width_uint::type; + + virtual __device__ __forceinline__ T operator()(const UINT_T value, [[maybe_unused]] const vi_t vector_index); +}; + +template +struct DecompressorBase { + /* Constructor, cannot be enforced + __device__ DecompressorBase(const ColumnT column, + const vi_t vector_index, const lane_t lane) + */ + + virtual void __device__ unpack_next_into(T* __restrict out); +}; + +template +struct BPColumn { + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + size_t n_vecs; + + UINT_T* packed_array; + vbw_t* bit_widths; + size_t* vector_offsets; +}; + +template +struct FFORColumn { + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + BPColumn bp; + UINT_T* bases; +}; + +template +struct ALPColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + FFORColumn ffor; + + INT_T* factors; + T* fractions; + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* counts; +}; + +template +struct ALPExtendedColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + FFORColumn ffor; + + INT_T* factors; + T* fractions; + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* offsets_counts; +}; +} // namespace device + +namespace host { + +template +struct BPColumn { + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::BPColumn; + + size_t n_values; + size_t n_packed_values; + + size_t get_n_values() const { + return n_values; + } + size_t get_n_vecs() const { + return utils::get_n_vecs_from_size(n_values); + } + + UINT_T* packed_array; + vbw_t* bit_widths; + size_t* vector_offsets; + + device::BPColumn copy_to_device() const { + const size_t branchless_extra_access_buffer = sizeof(T) * utils::get_n_lanes() * 4; + return device::BPColumn { + n_values, + get_n_vecs(), + GPUArray(n_packed_values, branchless_extra_access_buffer, packed_array).release(), + GPUArray(get_n_vecs(), bit_widths).release(), + GPUArray(get_n_vecs(), vector_offsets).release()}; + } +}; + +template +struct FFORColumn { + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::FFORColumn; + + BPColumn bp; + UINT_T* bases; + + size_t get_n_values() const { + return bp.n_values; + } + size_t get_n_vecs() const { + return bp.get_n_vecs(); + } + + device::FFORColumn copy_to_device() const { + return device::FFORColumn { + get_n_values(), bp.copy_to_device(), GPUArray(bp.get_n_vecs(), bases).release()}; + } +}; + +template +struct ALPExtendedColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::ALPExtendedColumn; + + FFORColumn ffor; + + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* offsets_counts; + + size_t compressed_size_bytes_alp_extended; + + size_t get_n_values() const { + return ffor.bp.n_values; + } + size_t get_n_vecs() const { + return ffor.bp.get_n_vecs(); + } + + double get_compression_ratio() const { + return static_cast(ffor.bp.n_values * sizeof(T)) / + static_cast(compressed_size_bytes_alp_extended); + } + + device::ALPExtendedColumn copy_to_device() const { + size_t branchless_and_prefetch_buffer = consts::MAX_UNPACK_N_VECS; + return device::ALPExtendedColumn { + get_n_values(), + ffor.copy_to_device(), + GPUArray(consts::as::FACT_ARR_COUNT, alp::Constants::FACT_ARR.data()).release(), + GPUArray(consts::as::FRAC_ARR_COUNT, alp::Constants::FRAC_ARR.data()).release(), + GPUArray(ffor.bp.get_n_vecs(), factor_indices).release(), + GPUArray(ffor.bp.get_n_vecs(), fraction_indices).release(), + n_exceptions, + GPUArray(ffor.bp.get_n_vecs(), exceptions_offsets).release(), + GPUArray(n_exceptions, branchless_and_prefetch_buffer, exceptions).release(), + GPUArray(n_exceptions, branchless_and_prefetch_buffer, positions).release(), + GPUArray(ffor.bp.get_n_vecs() * utils::get_n_lanes(), offsets_counts).release(), + }; + } +}; + +template +struct ALPColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::ALPColumn; + + FFORColumn ffor; + + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* counts; + + size_t compressed_size_bytes_alp; + size_t compressed_size_bytes_alp_extended; + + size_t get_n_values() const { + return ffor.bp.n_values; + } + size_t get_n_vecs() const { + return ffor.bp.get_n_vecs(); + } + + double get_compression_ratio() const { + return static_cast(ffor.bp.n_values * sizeof(T)) / static_cast(compressed_size_bytes_alp); + } + + device::ALPColumn copy_to_device() const { + return device::ALPColumn { + get_n_values(), + ffor.copy_to_device(), + GPUArray(consts::as::FACT_ARR_COUNT, alp::Constants::FACT_ARR.data()).release(), + GPUArray(consts::as::FRAC_ARR_COUNT, alp::Constants::FRAC_ARR.data()).release(), + GPUArray(ffor.bp.get_n_vecs(), factor_indices).release(), + GPUArray(ffor.bp.get_n_vecs(), fraction_indices).release(), + n_exceptions, + GPUArray(ffor.bp.get_n_vecs(), exceptions_offsets).release(), + GPUArray(n_exceptions, exceptions).release(), + GPUArray(n_exceptions, positions).release(), + GPUArray(ffor.bp.get_n_vecs(), counts).release(), + }; + } + + std::tuple convert_exceptions_to_lane_divided_format() const { + constexpr auto N_LANES = utils::get_n_lanes(); + constexpr auto VALUES_PER_LANE = utils::get_values_per_lane(); + + // New exception allocations + T* out_exceptions = reinterpret_cast(malloc(sizeof(T) * n_exceptions)); + uint16_t* out_positions = reinterpret_cast(malloc(sizeof(uint16_t) * n_exceptions)); + uint16_t* out_offsets_counts = + reinterpret_cast(malloc(sizeof(uint16_t) * ffor.get_n_vecs() * N_LANES)); + + // Intermediate arrays for reordering positions and exceptions + T vec_exceptions[consts::VALUES_PER_VECTOR]; + T vec_exceptions_positions[consts::VALUES_PER_VECTOR]; + uint16_t lane_counts[N_LANES]; + + // Copies of pointers for pointer arithmetic + T* c_exceptions = exceptions; + uint16_t* c_positions = positions; + T* c_out_exceptions = out_exceptions; + uint16_t* c_out_positions = out_positions; + uint16_t* c_out_offsets_counts = out_offsets_counts; + + for (size_t vec_index {0}; vec_index < ffor.get_n_vecs(); ++vec_index) { + uint32_t vec_exception_count = counts[vec_index]; + + // Reset counts + for (size_t j {0}; j < N_LANES; ++j) { + lane_counts[j] = 0; + } + + // Split all exceptions into lanes + for (size_t exception_index {0}; exception_index < vec_exception_count; ++exception_index) { + T exception = c_exceptions[exception_index]; + uint16_t position = c_positions[exception_index]; + + uint32_t lane = position % N_LANES; + uint32_t lane_exception_count = lane_counts[lane]; + ++lane_counts[lane]; + vec_exceptions[lane * VALUES_PER_LANE + lane_exception_count] = exception; + vec_exceptions_positions[lane * VALUES_PER_LANE + lane_exception_count] = position; + } + + // Merge and concatenate all exceptions per lane into single contiguous + // array + uint32_t vec_exceptions_counter = 0; + for (size_t lane {0}; lane < N_LANES; ++lane) { + uint32_t exc_in_lane_count = lane_counts[lane]; + for (size_t exc_in_lane {0}; exc_in_lane < exc_in_lane_count; ++exc_in_lane) { + + c_out_exceptions[vec_exceptions_counter] = vec_exceptions[lane * VALUES_PER_LANE + exc_in_lane]; + c_out_positions[vec_exceptions_counter] = + vec_exceptions_positions[lane * VALUES_PER_LANE + exc_in_lane]; + ++vec_exceptions_counter; + } + + c_out_offsets_counts[lane] = (exc_in_lane_count << 10) | (vec_exceptions_counter - exc_in_lane_count); + } + + c_exceptions += vec_exception_count; + c_positions += vec_exception_count; + c_out_exceptions += vec_exception_count; + c_out_positions += vec_exception_count; + c_out_offsets_counts += utils::get_n_lanes(); + } + + return std::make_tuple(out_exceptions, out_positions, out_offsets_counts); + } + + ALPExtendedColumn create_extended_column() const { + auto [e_exceptions, e_positions, e_offsets_counts] = convert_exceptions_to_lane_divided_format(); + return ALPExtendedColumn {FFORColumn { + BPColumn { + ffor.bp.n_values, + ffor.bp.n_packed_values, + utils::copy_array(ffor.bp.packed_array, ffor.bp.n_packed_values), + utils::copy_array(ffor.bp.bit_widths, get_n_vecs()), + utils::copy_array(ffor.bp.vector_offsets, get_n_vecs()), + }, + utils::copy_array(ffor.bases, get_n_vecs()), + }, + utils::copy_array(factor_indices, get_n_vecs()), + utils::copy_array(fraction_indices, get_n_vecs()), + n_exceptions, + utils::copy_array(exceptions_offsets, get_n_vecs()), + e_exceptions, + e_positions, + e_offsets_counts, + compressed_size_bytes_alp_extended}; + } +}; + +template +void free_column(BPColumn column) { + delete[] column.packed_array; + delete[] column.bit_widths; + delete[] column.vector_offsets; +} + +template +void free_column(FFORColumn column) { + free_column(column.bp); + delete[] column.bases; +} + +template +void free_column(ALPColumn column) { + free_column(column.ffor); + delete[] column.factor_indices; + delete[] column.fraction_indices; + delete[] column.exceptions_offsets; + delete[] column.exceptions; + delete[] column.positions; + delete[] column.counts; +} + +template +void free_column(ALPExtendedColumn column) { + free_column(column.ffor); + delete[] column.factor_indices; + delete[] column.fraction_indices; + delete[] column.exceptions_offsets; + delete[] column.exceptions; + delete[] column.positions; + delete[] column.offsets_counts; +} + +// Structs that are passed to the GPU cannot contain methods, +// that is why there is a separate method for the destructors +template +void free_column(device::BPColumn column) { + free_device_pointer(column.packed_array); + free_device_pointer(column.bit_widths); + free_device_pointer(column.vector_offsets); +} + +template +void free_column(device::FFORColumn column) { + free_column(column.bp); + free_device_pointer(column.bases); +} + +template +void free_column(device::ALPColumn column) { + free_column(column.ffor); + free_device_pointer(column.factors); + free_device_pointer(column.fractions); + free_device_pointer(column.factor_indices); + free_device_pointer(column.fraction_indices); + free_device_pointer(column.exceptions_offsets); + free_device_pointer(column.exceptions); + free_device_pointer(column.positions); + free_device_pointer(column.counts); +} + +template +void free_column(device::ALPExtendedColumn column) { + free_column(column.ffor); + free_device_pointer(column.factors); + free_device_pointer(column.fractions); + free_device_pointer(column.factor_indices); + free_device_pointer(column.fraction_indices); + free_device_pointer(column.exceptions_offsets); + free_device_pointer(column.exceptions); + free_device_pointer(column.positions); + free_device_pointer(column.offsets_counts); +} + +} // namespace host +} // namespace flsgpu + +#endif // STRUCTS_CUH diff --git a/galp/src/include/flsgpu/utils.cuh b/galp/src/include/flsgpu/utils.cuh new file mode 100644 index 00000000..e44e90ba --- /dev/null +++ b/galp/src/include/flsgpu/utils.cuh @@ -0,0 +1,104 @@ +#ifndef FASTLANES_UTILS_H +#define FASTLANES_UTILS_H + +#include "consts.cuh" +#include +#include +#include +#include +#include + +namespace utils { // internal functions + // +template +struct same_width_int { + using type = typename std::conditional< + sizeof(T) == 8, + int64_t, + typename std::conditional::type>::type>::type; +}; + +template +struct same_width_uint { + using type = typename std::conditional< + sizeof(T) == 8, + uint64_t, + typename std::conditional::type>::type>::type; +}; + +template +constexpr returnT sizeof_in_bits() { + return sizeof(T) * 8; +} + +template +constexpr T min(const T& a, const T& b) { + // C++-11 and older do not offer constexpr, that is why this is added here + return a <= b ? a : b; +} + +template +constexpr T_out reinterpret_type(const T_in& in) { + // Type punning, the compiler optimizes out the memcpy + // https://cuda.godbolt.org/z/bKG89YTY7 + // https://stackoverflow.com/questions/47037104/cuda-type-punning-memcpy-vs-ub-union + // T_out out; + // T_in staged = in; + // memcpy(&out, &staged, min(sizeof(T_in), sizeof(T_out))); + // return out; + T_in staged = in; + return *reinterpret_cast(&staged); +} + +template +constexpr T h_set_first_n_bits(const int32_t count) { + return (count < sizeof_in_bits() ? static_cast((T {1} << int32_t {count}) - T {1}) : static_cast(~T {0})); +} + +template +constexpr T set_first_n_bits(const int32_t count) { + using UINT_T = typename same_width_uint::type; + static_assert(std::is_integral::value, "T must be an integer type"); + + return reinterpret_type(std::numeric_limits::max() >> + reinterpret_type(utils::sizeof_in_bits() - count)); +} + +template +constexpr int32_t get_lane_bitwidth() { + return sizeof_in_bits(); +} + +template +constexpr int32_t get_n_lanes() { + return consts::REGISTER_WIDTH / get_lane_bitwidth(); +} + +template +constexpr int32_t get_values_per_lane() { + return consts::VALUES_PER_VECTOR / get_n_lanes(); +} + +template +constexpr int32_t get_compressed_vector_size(int32_t value_bit_width) { + return (consts::VALUES_PER_VECTOR * value_bit_width) / sizeof_in_bits(); +} + +constexpr size_t get_n_vecs_from_size(const size_t size) { + return (size + consts::VALUES_PER_VECTOR - 1) / consts::VALUES_PER_VECTOR; +} + +template +T* copy_array(const T* in, const size_t n_elements) { + T* out = new T[n_elements]; + std::memcpy(out, in, sizeof(T) * n_elements); + return out; +} + +} // namespace utils + +#endif // FASTLANES_UTILS_H diff --git a/galp/src/include/generator/generate_binaries.hpp b/galp/src/include/generator/generate_binaries.hpp new file mode 100644 index 00000000..816023b9 --- /dev/null +++ b/galp/src/include/generator/generate_binaries.hpp @@ -0,0 +1,116 @@ +// ============================================================================= +// generate_binaries.hpp ― Random-data generator + directory scanner (host-only) +// +// * **Now writes little-endian** (the de-facto standard on x86/ARM). +// * Automatically swaps bytes on big-endian hosts. +// * Produces float32 + float64 binaries and can scan data/floats + data/doubles. +// * Wrapped in host-only guards so NVCC’s device pass never sees . +// ============================================================================= +#ifndef GENERATE_BINARIES_HPP +#define GENERATE_BINARIES_HPP + +#include +#include +#include +#include +#include +#include +#include + +// ────────────────────────────────────────────────────────────────────────────── +// Host-only implementation (skipped entirely for __CUDA_ARCH__) +// ────────────────────────────────────────────────────────────────────────────── +namespace bin { +namespace detail { + +// compile-time endian test (uses compiler macros, no UB in constexpr) +inline constexpr bool is_little_endian = +#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + true; +#else + false; +#endif +; + +} // namespace detail + +// ----------------------------------------------------------------------------- +// write_little_endian ― write POD value in **little-endian** order +// ----------------------------------------------------------------------------- +template +inline void write_little_endian(std::ostream& os, const T& value) { + static_assert(std::is_trivially_copyable_v, "write_little_endian requires a trivially-copyable type"); + T tmp = value; + if constexpr (!detail::is_little_endian) { // host is big-endian + auto* p = reinterpret_cast(&tmp); + std::reverse(p, p + sizeof(T)); // swap to LE + } + os.write(reinterpret_cast(&tmp), sizeof(T)); +} + +// ----------------------------------------------------------------------------- +// generate_and_write ― write TOTAL random numbers (0-100, two decimals) +// to (float32 LE) and (float64 LE) +// returns the first HEAD_COUNT values +// ----------------------------------------------------------------------------- +inline std::vector +generate_and_write(const std::string& out32, const std::string& out64, size_t total, size_t head_count) { + std::mt19937_64 rng(std::random_device {}()); + std::uniform_real_distribution dist(0.0, 100.0); + + std::vector head; + head.reserve(head_count); + + std::ofstream f32(out32, std::ios::binary); + std::ofstream f64(out64, std::ios::binary); + + for (size_t i = 0; i < total; ++i) { + double val = std::round(dist(rng) * 100.0) / 100.0; + if (head.size() < head_count) + head.push_back(val); + + write_little_endian(f32, static_cast(val)); + write_little_endian(f64, val); + } + return head; +} + +// ----------------------------------------------------------------------------- +// generate_write_and_scan ― one-stop helper: +// 1) writes /1.bin + /1.bin (little-endian) +// 2) scans both directories for every *.bin +// 3) returns first HEAD_COUNT values + sorted file lists +// ----------------------------------------------------------------------------- +struct GenResult { + std::vector head; // first N random values + std::vector float_files; // *.bin under floats_dir + std::vector double_files; // *.bin under doubles_dir +}; + +inline GenResult generate_write_and_scan(const std::filesystem::path& floats_dir, + const std::filesystem::path& doubles_dir, + size_t total, + size_t head_count) { + GenResult res; + + // (1) produce "1.bin" in each directory + res.head = generate_and_write( + (floats_dir / "random.bin").string(), (doubles_dir / "random.bin").string(), total, head_count); + + // (2) collect existing *.bin files + auto collect = [](const std::filesystem::path& dir, std::vector& out) { + if (!std::filesystem::exists(dir)) + return; + for (const auto& e : std::filesystem::directory_iterator(dir)) + if (e.is_regular_file() && e.path().extension() == ".bin") + out.emplace_back(e.path().string()); + std::sort(out.begin(), out.end()); + }; + collect(floats_dir, res.float_files); + collect(doubles_dir, res.double_files); + + return res; +} + +} // namespace bin +#endif // !__CUDA_ARCH__ diff --git a/galp/src/include/nvcomp/benchmark-compressors.cuh b/galp/src/include/nvcomp/benchmark-compressors.cuh new file mode 100644 index 00000000..a1087f31 --- /dev/null +++ b/galp/src/include/nvcomp/benchmark-compressors.cuh @@ -0,0 +1,141 @@ +#include "../engine/enums.cuh" +#include "../flsgpu/flsgpu-api.cuh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef BENCHMARK_COMPRESSORS_CUH +#define BENCHMARK_COMPRESSORS_CUH + +template +bool check_if_device_buffers_are_equal(const T* a, const T* b, const size_t n_values) { + // Convert to uint8_t as we don't want to compare floats (-nan == -nan => + // false) + thrust::device_ptr d_a(reinterpret_cast(const_cast(a))); + thrust::device_ptr d_b(reinterpret_cast(const_cast(b))); + + return thrust::equal(d_a, d_a + n_values, d_b); +} + +class CudaStopwatch { +private: + cudaEvent_t startEvent, stopEvent; + cudaStream_t stream; + float milliseconds = 0.0f; + +public: + CudaStopwatch(cudaStream_t stream = 0) + : stream(stream) { + CUDA_SAFE_CALL(cudaEventCreate(&startEvent)); + CUDA_SAFE_CALL(cudaEventCreate(&stopEvent)); + } + + ~CudaStopwatch() { + CUDA_SAFE_CALL(cudaEventDestroy(startEvent)); + CUDA_SAFE_CALL(cudaEventDestroy(stopEvent)); + } + + void start() { + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + CUDA_SAFE_CALL(cudaEventRecord(startEvent, stream)); + } + + double stop() { + CUDA_SAFE_CALL(cudaEventRecord(stopEvent, stream)); + CUDA_SAFE_CALL(cudaEventSynchronize(stopEvent)); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + + CUDA_SAFE_CALL(cudaEventElapsedTime(&milliseconds, startEvent, stopEvent)); + return milliseconds; + } + + double get_result() { + return milliseconds; + } +}; + +struct BenchmarkResult { + bool results_match; + double execution_time_ms; + double compression_ratio; + + void log_result(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const size_t n_bytes, + const std::string data_name) const { + + const char* is_valid = results_match ? "valid" : "wrong"; + + printf("%s,%s,%s,%s,%lu,%f,%f\n", + enums_nvcomp::comparison_type_to_string(comparison_type).c_str(), + enums_nvcomp::compression_type_to_string(compression_type).c_str(), + data_name.c_str(), + is_valid, + n_bytes, + execution_time_ms, + compression_ratio); + } +}; + +template +struct is_equal_to { + T value; + is_equal_to(T value) + : value(value) { + } + __host__ __device__ bool operator()(T x) { + return x == value; + } +}; + +template +BenchmarkResult benchmark_thrust(const T* input, const size_t value_count, const T value_to_search_for); + +template +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const T* input, + const size_t value_count, + const T value_to_search_for); + +template +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const T* input, + const size_t value_count, + const T value_to_search_for); + +BenchmarkResult benchmark_thrust(const float* input, const size_t value_count, const float value_to_search_for); +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const float* input, + const size_t value_count, + const float value_to_search_for); +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const float* input, + const size_t value_count, + const float value_to_search_for); + +BenchmarkResult benchmark_thrust(const double* input, const size_t value_count, const double value_to_search_for); +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const double* input, + const size_t value_count, + const double value_to_search_for); +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const double* input, + const size_t value_count, + const double value_to_search_for); + +#endif // BENCHMARK_COMPRESSORS_CUH diff --git a/galp/src/include/nvcomp/nvcomp-compressors.cuh b/galp/src/include/nvcomp/nvcomp-compressors.cuh new file mode 100644 index 00000000..f5691801 --- /dev/null +++ b/galp/src/include/nvcomp/nvcomp-compressors.cuh @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifndef NVCOMP_COMPRESSORS_H +#define NVCOMP_COMPRESSORS_H + +#include "engine/enums.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include "nvcomp.h" +#include "nvcomp.hpp" +#include "nvcomp/benchmark-compressors.cuh" +#include "nvcomp/deflate.h" +#include "nvcomp/deflate.hpp" +#include "nvcomp/gdeflate.h" +#include "nvcomp/gdeflate.hpp" +#include "nvcomp/lz4.hpp" +#include "nvcomp/nvcompManager.hpp" +#include "nvcomp/nvcompManagerFactory.hpp" +#include "nvcomp/zstd.h" +#include "nvcomp/zstd.hpp" + +namespace hwc { +using nvcompCompressionManager = nvcomp::PimplManager; + +nvcompCompressionManager* get_compressor_manager(const enums_nvcomp::CompressionType compression_type, + const nvcompType_t data_type = NVCOMP_TYPE_CHAR, + const size_t chunk_size = 1 << 16); + +struct CompressedBuffer { + nvcomp::CompressionConfig compression_config; + uint8_t* compressed_buffer; + size_t decompressed_size; + size_t compressed_size; + + CompressedBuffer(nvcomp::CompressionConfig compression_config) + : compression_config(compression_config) { + + CUDA_SAFE_CALL(cudaMalloc(&compressed_buffer, compression_config.max_compressed_buffer_size)); + } + + double get_compression_ratio() { + return static_cast(decompressed_size) / static_cast(compressed_size); + } + + double get_allocation_compression_ratio() { + return static_cast(compression_config.max_compressed_buffer_size) / + static_cast(compression_config.uncompressed_buffer_size); + } + + // Manual resource freeing + void free() { + CUDA_SAFE_CALL(cudaFree(compressed_buffer)); + } +}; + +struct Compressor { + nvcompCompressionManager* manager; + + Compressor(const enums_nvcomp::CompressionType compression_type) { + manager = get_compressor_manager(compression_type); + } + + CompressedBuffer compress(uint8_t* input_buffer, const size_t input_buffer_len) { + CompressedBuffer compressed_buffer(manager->configure_compression(input_buffer_len)); + manager->compress(input_buffer, compressed_buffer.compressed_buffer, compressed_buffer.compression_config); + compressed_buffer.decompressed_size = input_buffer_len; + compressed_buffer.compressed_size = manager->get_compressed_output_size(compressed_buffer.compressed_buffer); + return compressed_buffer; + } + + uint8_t* decompress(const CompressedBuffer compressed_buffer, CudaStopwatch& stopwatch) { + nvcomp::DecompressionConfig decomp_config = + manager->configure_decompression(compressed_buffer.compressed_buffer); + uint8_t* decompressed_buffer; + CUDA_SAFE_CALL(cudaMalloc(&decompressed_buffer, decomp_config.decomp_data_size)); + + stopwatch.start(); + manager->decompress(decompressed_buffer, compressed_buffer.compressed_buffer, decomp_config); + stopwatch.stop(); + + return decompressed_buffer; + } + + // Manual resource freeing + void free() { + delete manager; + } +}; + +__global__ void check_buffer_equality(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length, bool* out); + +bool compare_d_buffers(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length); + +template +struct DummyColumn { + T* in; + size_t n_values; +}; + +template +struct DummyDecompressor : flsgpu::device::DecompressorBase { + using UINT_T = typename utils::same_width_uint::type; + flsgpu::device::BitUnpackerDummy> unpacker; + + __device__ __forceinline__ + DummyDecompressor(const DummyColumn column, const vi_t vector_index, const lane_t lane) + : unpacker(reinterpret_cast(column.in + vector_index * consts::VALUES_PER_VECTOR), + lane, + utils::sizeof_in_bits(), + flsgpu::device::BPFunctor()) { + } + + void __device__ unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + } +}; + +} // namespace hwc + +#endif // NVCOMP_COMPRESSORS_H diff --git a/galp/test/CMakeLists.txt b/galp/test/CMakeLists.txt new file mode 100644 index 00000000..68560107 --- /dev/null +++ b/galp/test/CMakeLists.txt @@ -0,0 +1,18 @@ +add_executable(galp_tests + thrust_memcpy_test.cu + thrust_core_test.cu + galp_test.cu +) + + +target_link_libraries(galp_tests PRIVATE + galp_core + GTest::gtest_main + FastLanes::core +) + +apply_warnings(galp_tests) + +gtest_discover_tests(galp_tests + DISCOVERY_TIMEOUT 60 +) \ No newline at end of file diff --git a/galp/test/galp_test.cu b/galp/test/galp_test.cu new file mode 100644 index 00000000..8a8fb7cb --- /dev/null +++ b/galp/test/galp_test.cu @@ -0,0 +1,232 @@ +#include "alp/alp-bindings.cuh" +#include "data/fastlanes_data.hpp" +#include "engine/device-utils.cuh" +#include "engine/kernels.cuh" +#include "flsgpu/alp.cuh" +#include "flsgpu/structs.cuh" +#include "generator/generate_binaries.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for std::pair +#include + +template +std::vector read_file(const std::string& path) { + // Open file in binary mode, positioned at end to get its size + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error("Could not open file: " + path); + } + + // Determine file size in bytes + std::streamsize bytes = file.tellg(); + if (bytes < 0) { + throw std::runtime_error("Could not determine file size: " + path); + } + + // Ensure the file contains an integral number of T elements + if (bytes % sizeof(T) != 0) { + throw std::runtime_error("File size (" + std::to_string(bytes) + " bytes) is not a multiple of element size (" + + std::to_string(sizeof(T)) + " bytes)"); + } + + // Calculate number of elements + std::size_t count = static_cast(bytes / sizeof(T)); + + // Seek back to beginning and read all data into a vector + file.seekg(0, std::ios::beg); + std::vector data(count); + if (!file.read(reinterpret_cast(data.data()), bytes)) { + throw std::runtime_error("Error reading file: " + path); + } + + return data; +} + +// ----------------------------------------------------------------------------- +// CSV loader – light‑weight, header‑only +// ----------------------------------------------------------------------------- +template +std::vector read_csv(const std::filesystem::path& path) { + std::ifstream file(path); + if (!file) { + throw std::runtime_error("Could not open csv file: " + path.string()); + } + + std::vector data; + std::string line; + while (std::getline(file, line)) { + std::stringstream ss(line); + std::string cell; + while (std::getline(ss, cell, ',')) { + if (!cell.empty()) { + data.push_back(static_cast(std::stod(cell))); + } + } + } + return data; +} + +template +bool check_if_device_buffers_are_equal(const T* a, const T* b, const size_t n_values) { + // Convert to uint8_t as we don't want to compare floats (-nan == -nan => + // false) + thrust::device_ptr d_a(reinterpret_cast(const_cast(a))); + thrust::device_ptr d_b(reinterpret_cast(const_cast(b))); + + return thrust::equal(d_a, d_a + n_values, d_b); +} + +template +using ALPDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>; + +template +using ALPExtendedDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>; + +struct CLIArgs; +template +inline void test_alp(const std::filesystem::path& path) { + auto data_vec = read_file(path); + + flsgpu::host::ALPColumn column = alp::encode(data_vec.data(), data_vec.size(), true); + GPUArray d_decompression_result(data_vec.size()); + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + + flsgpu::device::ALPColumn d_column = column.copy_to_device(); + kernels::device:: + decompress_column, flsgpu::device::ALPColumn> + <<>>(d_column, d_decompression_result.get()); + + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + flsgpu::host::free_column(d_column); + + bool kernel_successful = false; + GPUArray d_input(data_vec.size(), data_vec.data()); + + kernel_successful = + check_if_device_buffers_are_equal(d_decompression_result.get(), d_input.get(), column.get_n_values()); + + EXPECT_TRUE(kernel_successful); + + flsgpu::host::free_column(column); +} + +struct CLIArgs; +template +inline void test_galp(const std::filesystem::path& path) { + auto data_vec = read_file(path); + + flsgpu::host::ALPColumn column = alp::encode(data_vec.data(), data_vec.size(), true); + flsgpu::host::ALPExtendedColumn column_extended = column.create_extended_column(); + flsgpu::device::ALPExtendedColumn d_column = column_extended.copy_to_device(); + GPUArray d_decompression_result(data_vec.size()); + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + kernels::device::decompress_column, + flsgpu::device::ALPExtendedColumn> + <<>>(d_column, d_decompression_result.get()); + + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + double compression_ratio = column_extended.get_compression_ratio(); + std::cout << "compression_ratio : " << compression_ratio << std::endl; + flsgpu::host::free_column(column_extended); + flsgpu::host::free_column(d_column); +} + +// ----------------------------------------------------------------------------- +// Exact copy of test_galp<…>, but uses read_csv instead of read_file +// ----------------------------------------------------------------------------- +template +inline void test_galp_csv(const std::filesystem::path& path) { + auto data_vec = read_csv(path); + + flsgpu::host::ALPColumn column = alp::encode(data_vec.data(), data_vec.size(), true); + flsgpu::host::ALPExtendedColumn column_extended = column.create_extended_column(); + flsgpu::device::ALPExtendedColumn d_column = column_extended.copy_to_device(); + + GPUArray d_decompression_result(data_vec.size()); + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + + kernels::device::decompress_column, + flsgpu::device::ALPExtendedColumn> + <<>>(d_column, d_decompression_result.get()); + + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + double compression_ratio = column_extended.get_compression_ratio(); + std::cout << "compression_ratio : " << compression_ratio << std::endl; + + flsgpu::host::free_column(column_extended); + flsgpu::host::free_column(d_column); +} + +TEST(ALP, TEST_ALP) { + // ------------------------------------------------------------------------- + // (0) Generate new binaries + discover directories + // ------------------------------------------------------------------------- + const size_t TOTAL = 25'600 * 1'024; + const size_t HEAD = 0; + + const std::filesystem::path floats_dir = FLS_GALP_SOURCE_DIR "/data/floats"; + const std::filesystem::path doubles_dir = FLS_GALP_SOURCE_DIR "/data/doubles"; + + bin::GenResult gen = bin::generate_write_and_scan(floats_dir, doubles_dir, TOTAL, HEAD); + for (const auto& path : gen.float_files) { + test_alp(path); + } +} + +TEST(GALP, TEST_GALP) { + // ------------------------------------------------------------------------- + // (0) Generate new binaries + discover directories + // ------------------------------------------------------------------------- + const size_t TOTAL = 25'600 * 1'024; + const size_t HEAD = 0; + + const std::filesystem::path floats_dir = FLS_GALP_SOURCE_DIR "/data/floats"; + const std::filesystem::path doubles_dir = FLS_GALP_SOURCE_DIR "/data/doubles"; + + bin::GenResult gen = bin::generate_write_and_scan(floats_dir, doubles_dir, TOTAL, HEAD); + for (const auto& path : gen.float_files) { + test_galp(path); + } +} + +TEST(GALP, TEST_GALP_BY_GALP_DATASET) { + namespace fs = std::filesystem; + + for (const auto& [name, dir] : fastlanes::galp::dataset) { + for (const auto& entry : fs::directory_iterator(dir)) { + if (entry.is_regular_file() && entry.path().extension() == ".csv") { + std::cout << "Testing file: " << entry.path().string() << std::endl; + test_galp_csv(entry.path()); + } + } + } +} diff --git a/galp/test/thrust_core_test.cu b/galp/test/thrust_core_test.cu new file mode 100644 index 00000000..0789f93c --- /dev/null +++ b/galp/test/thrust_core_test.cu @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// -------------------------------------------------------------------- +// Custom functor for zip transform +// -------------------------------------------------------------------- +struct SumTuple { + __host__ __device__ int operator()(const thrust::tuple& t) const { + return thrust::get<0>(t) + thrust::get<1>(t); + } +}; + +// -------------------------------------------------------------------- +// 1. Containers & element-wise algorithms +// -------------------------------------------------------------------- +TEST(ThrustCore, TransformAdd) { + const int N = 128; + thrust::device_vector a(N); + thrust::device_vector b(N); + thrust::device_vector c(N); + + thrust::sequence(a.begin(), a.end(), 0); // 0..N-1 + thrust::fill(b.begin(), b.end(), 1); // all 1 + + thrust::transform(a.begin(), a.end(), b.begin(), c.begin(), thrust::plus()); + + thrust::host_vector h(c); + for (int i = 0; i < N; ++i) + EXPECT_EQ(h[i], i + 1); +} + +// -------------------------------------------------------------------- +// 2. Reductions & scans +// -------------------------------------------------------------------- +TEST(ThrustCore, ReduceAndInclusiveScan) { + const int N = 256; + thrust::device_vector d(N); + thrust::sequence(d.begin(), d.end(), 1); // 1..N + + int gpu_sum = thrust::reduce(d.begin(), d.end()); + int ref_sum = N * (N + 1) / 2; + EXPECT_EQ(gpu_sum, ref_sum); + + thrust::inclusive_scan(d.begin(), d.end(), d.begin()); + + int last = d.back(); + EXPECT_EQ(last, ref_sum); +} + +// -------------------------------------------------------------------- +// 3. Sorting & uniquing +// -------------------------------------------------------------------- +TEST(ThrustCore, SortAndUnique) { + std::vector temp = {5, 1, 4, 4, 3, 2, 1}; + thrust::host_vector h(temp.begin(), temp.end()); + thrust::device_vector d = h; + + thrust::sort(d.begin(), d.end()); + auto new_end = thrust::unique(d.begin(), d.end()); + + EXPECT_EQ(new_end - d.begin(), 5); + + thrust::host_vector out(d.begin(), new_end); + std::vector ref = {1, 2, 3, 4, 5}; + EXPECT_TRUE(std::equal(out.begin(), out.end(), ref.begin())); +} + +// -------------------------------------------------------------------- +// 4. Zip iterator, counting iterator & transform iterator +// -------------------------------------------------------------------- +TEST(ThrustCore, CountingAndZipIter) { + const int N = 64; + auto first = thrust::make_counting_iterator(0); + auto second = thrust::make_counting_iterator(0); + + thrust::device_vector d(N); + + thrust::transform(thrust::make_zip_iterator(thrust::make_tuple(first, second)), + thrust::make_zip_iterator(thrust::make_tuple(first + N, second + N)), + d.begin(), + SumTuple()); + + thrust::host_vector h = d; + for (int i = 0; i < N; ++i) + EXPECT_EQ(h[i], i + i); +} + +// -------------------------------------------------------------------- +// 5. Set difference (requires sorted ranges) +// -------------------------------------------------------------------- +TEST(ThrustCore, SetDifference) { + std::vector tempA = {0, 1, 2, 3, 4, 5}; + std::vector tempB = {1, 3, 5}; + + thrust::device_vector A(tempA.begin(), tempA.end()); + thrust::device_vector B(tempB.begin(), tempB.end()); + thrust::device_vector diff(A.size()); + + auto end = thrust::set_difference(A.begin(), A.end(), B.begin(), B.end(), diff.begin()); + + thrust::host_vector h(diff.begin(), end); + std::vector ref = {0, 2, 4}; + EXPECT_TRUE(std::equal(h.begin(), h.end(), ref.begin())); +} diff --git a/galp/test/thrust_memcpy_test.cu b/galp/test/thrust_memcpy_test.cu new file mode 100644 index 00000000..11e733b4 --- /dev/null +++ b/galp/test/thrust_memcpy_test.cu @@ -0,0 +1,25 @@ +// test/test_thrust_memcpy.cu + +#include +#include +#include +#include +#include + +TEST(ThrustMemcpy, HostToDeviceAndBack) { + constexpr std::size_t N = 1 << 12; // 4096 ints + + // 1) Host data: 0,1,2,…,N-1 + thrust::host_vector h_src(N); + thrust::sequence(h_src.begin(), h_src.end()); + + // 2) Copy to device (H→D constructor) + thrust::device_vector d_buf = h_src; + + // 3) Copy back to host (D→H constructor) + thrust::host_vector h_dst = d_buf; + + // 4) Verify equality with a single call + bool same = thrust::equal(h_src.begin(), h_src.end(), h_dst.begin()); + EXPECT_TRUE(same) << "Data changed after GPU round-trip"; +} diff --git a/gpu/CMakeLists.txt b/gpu/CMakeLists.txt deleted file mode 100644 index 8b137891..00000000 --- a/gpu/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/gpu/cuda/CMakeLists.txt b/gpu/cuda/CMakeLists.txt deleted file mode 100644 index 8b137891..00000000 --- a/gpu/cuda/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/mk/benchmark.mk b/mk/benchmark.mk new file mode 100644 index 00000000..d390eba3 --- /dev/null +++ b/mk/benchmark.mk @@ -0,0 +1,34 @@ +# mk/benchmark.mk +# ──────────────── +# Runs benchmark/result/galp/plot.py from *inside* its own folder, +# so the script can see the neighbouring patch.csv file. + +# ── Virtual‑env (absolute so it still works after any cd) +VENV_DIR := $(abspath ../.venv) + +PYTHON3 := $(VENV_DIR)/bin/python3 +PYTHON := $(if $(wildcard $(PYTHON3)),$(PYTHON3),$(VENV_DIR)/bin/python) +PIP := $(PYTHON) -m pip + +# ── Script location (relative to project root) +PY_SCRIPT := benchmark/result/galp/plot.py +PY_SCRIPT_DIR := $(dir $(PY_SCRIPT)) # benchmark/result/galp/ +PY_SCRIPT_NAME := $(notdir $(PY_SCRIPT)) # plot.py + +.PHONY: venv plot clean-venv + +venv: + @echo "Creating virtualenv in $(VENV_DIR)…" + @test -d $(VENV_DIR) || python3 -m venv $(VENV_DIR) + @echo "Installing deps into venv…" + @$(PIP) install --upgrade pip pandas matplotlib + +plot: venv + @echo "Plotting from patch.csv…" + # 1) cd to project root 2) cd into script’s folder 3) run script + @cd .. && cd $(PY_SCRIPT_DIR) && $(PYTHON) $(PY_SCRIPT_NAME) + @echo "Done." + +clean-venv: + @rm -rf $(VENV_DIR) + @echo "Removed venv." diff --git a/mk/format.mk b/mk/format.mk index 9c937499..d3d0b064 100644 --- a/mk/format.mk +++ b/mk/format.mk @@ -1,6 +1,6 @@ IMAGE := ghcr.io/azimafroozeh/clang-format-python/clang-format-python:14 UBUNTU_IMAGE := ubuntu:22.04 -DIRS := examples src benchmark test data/include python +DIRS := examples src benchmark test data/include python cuda galp EXCLUDES := --exclude src/include/fls/json/nlohmann \ --exclude src/include/flatbuffers \ --exclude src/primitives \ @@ -14,8 +14,10 @@ DOCKER := docker run --rm -v "$$(pwd)":/app -w /app # the actual clang-format invocation FMT_CMD := python3 scripts/run-clang-format.py \ -r $(DIRS) \ + --extensions='c,h,C,H,cpp,hpp,cc,hh,cxx,hxx,cu,cuh' \ $(EXCLUDES) + .PHONY: format clang-format format-check rust-format-check clang-format: diff --git a/scripts/generate_synthetic_data.py b/scripts/generate_synthetic_data.py index 43277d95..49e2b369 100644 --- a/scripts/generate_synthetic_data.py +++ b/scripts/generate_synthetic_data.py @@ -18,7 +18,7 @@ from generator_helpers.boolean_generator import * from generator_helpers.i08_generator import * from generator_helpers.u08_generator import * - +from generator_helpers.galp_generator import * # --------------------------- # Faker Providers @@ -622,6 +622,7 @@ def main(): generate_any_value_count() generate_subnormals() generate_fsst_test() + fls_galp() logging.info("✅ All data files generated successfully.") # ← NEW diff --git a/scripts/generator_helpers/galp_generator.py b/scripts/generator_helpers/galp_generator.py new file mode 100644 index 00000000..f6606eaf --- /dev/null +++ b/scripts/generator_helpers/galp_generator.py @@ -0,0 +1,76 @@ +""" +Module: scripts/generate_helpers/galp_generator.py +Description: For three fields—one random two‑decimal GALP, one constant 1.00, + and one POSITIVE_INF—writes each as its own single‑column CSV + + schema.json, in both ROW_GROUP_SIZE and VEC_SIZE variants, + and for both FLOAT and DOUBLE schema types. +""" + +import random +from decimal import Decimal, ROUND_HALF_UP +from pathlib import Path +import json + +from .write_helpers import write_csv +from .common import ROW_GROUP_SIZE, VEC_SIZE + + +def _gen_galp(_faker, row_id): + """Random number in [0.0,100.0], two decimals.""" + raw = random.uniform(0.0, 100.0) + return [ + float( + Decimal(str(raw)) + .quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) + ) + ] + + +def _gen_constant(_faker, row_id): + """Constant = 1.00.""" + return [1.00] + + +def _gen_inf(_faker, row_id): + """Constant = +infinity.""" + return [float("inf")] + + +def fls_galp(): + """ + For each of three fields—GALP, CONSTANT, POSITIVE_INF—and for each precision + (FLOAT vs DOUBLE), write: + + data/generated/galp///single_column/generated.csv (ROW_GROUP_SIZE rows) + data/generated/galp///single_column/schema.json + + data/generated/galp///one_vector/generated.csv (VEC_SIZE rows) + data/generated/galp///one_vector/schema.json + """ + base_dir = Path.cwd() / "../data" / "generated" / "galp" + + # the three fields we want to generate + tasks = [ + ("galp", "SYNTHETIC_DATA_GALP", _gen_galp), + ("constant", "SYNTHETIC_DATA_CONSTANT", _gen_constant), + ("positive_inf", "SYNTHETIC_DATA_POSITIVE_INF", _gen_inf), + ] + + # emit both FLOAT and DOUBLE versions + precisions = [ + ("float", "FLOAT"), + ("double", "DOUBLE"), + ] + + for prec_key, prec_type in precisions: + for field_key, column_name, gen_fn in tasks: + # 1) Row‑group (single_column) + dir_rg = base_dir / prec_key / field_key / "single_column" + write_csv(dir_rg, gen_fn, ROW_GROUP_SIZE) + schema = {"columns": [{"name": column_name, "type": prec_type}]} + (dir_rg / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") + + # 2) One‑vector version + dir_ov = base_dir / prec_key / field_key / "one_vector" + write_csv(dir_ov, gen_fn, VEC_SIZE) + (dir_ov / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") diff --git a/src/connection.cpp b/src/connection.cpp index 974e04f0..9f5b3708 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -54,18 +54,23 @@ up Connection::read_fls(const path& file_path) { return make_unique(file_path, *this); } -void prepare_rowgroup(Rowgroup& rowgroup) { - - // could be combined +void prepare_rowgroup(Rowgroup& rowgroup, const Config& config) { + // init rowgroup.Init(); - rowgroup.Cast(); + + // Only cast if schema wasn’t forced + const bool shouldCast = !config.is_forced_schema && !config.is_forced_schema_pool; + if (shouldCast) { + rowgroup.Cast(); + } + rowgroup.Finalize(); rowgroup.GetStatistics(); } void Connection::prepare_table() const { for (auto& rowgroup : m_table->m_rowgroups) { - prepare_rowgroup(*rowgroup); + prepare_rowgroup(*rowgroup, *m_config); } } diff --git a/src/cor/compression_plan_history.cpp b/src/cor/compression_plan_history.cpp deleted file mode 100644 index fc0b2ff0..00000000 --- a/src/cor/compression_plan_history.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "fls/cor/compression_plan_history.hpp" -#include "fls/cor/compression_plan.hpp" -#include - -namespace fast_lanes { -void History::Save(sp debug_sp) { /* NOLINT*/ - m_plan_sp_vec.push_back(debug_sp); -} - -void History::Last(std::ostream& out) { - m_plan_sp_vec.back()->ToCsv(out); -} -void History::ToTable(TxtTab& txt_tab) { - m_plan_sp_vec.back()->ToTable(txt_tab); -} -} // namespace fast_lanes diff --git a/src/cor/cpu.cpp b/src/cor/cpu.cpp deleted file mode 100644 index 3e2b39da..00000000 --- a/src/cor/cpu.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "fls/cor/cpu.hpp" -namespace fast_lanes { -} // namespace fast_lanes diff --git a/src/cor/lyt/col.cpp b/src/cor/lyt/col.cpp deleted file mode 100644 index 4274c8f7..00000000 --- a/src/cor/lyt/col.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include "fls/cor/lyt/col.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/io/external_memory.hpp" - -namespace fastlanes { -template -Col::Col() - : m_count {0} { - m_buffer.resize(CFG::ROW_GROUP_SIZE); -} - -template -T* Col::operator[](n_t i) { - if constexpr (std::is_same()) { - FLS_ABORT("Not Supported") - } else { - FLS_ASSERT(i >= 0, "negative index.", " "); - FLS_ASSERT(m_count != 0, "0 size.", " "); - FLS_ASSERT(i < m_count, "Out of range index.", " "); - - return &(m_buffer[i]); - } - return nullptr; -} - -template -void Col::Share(T* data, n_t c) { -} - -template -void Col::Insert(T* data, n_t c) { - if constexpr (std::is_same()) { - FLS_ABORT("Not Supported") - } else { - FLS_ASSERT(m_count + c <= CFG::ROW_GROUP_SIZE, " ", "over_capacity"); - - ExternalMemory::Copy(m_buffer.data() + m_count, data, c * sizeof(T)); - m_count += c; - } -} - -template -T* Col::Data() { - // return m_buffer.data(); - return nullptr; -} - -template -n_t Col::Count() const { - return m_count; -} - -template -byte_c Col::Size() const { - return byte_c {sizeof(T) * m_buffer.size()}; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(Col) -} // namespace fastlanes diff --git a/src/cor/lyt/dic/CMakeLists.txt b/src/cor/lyt/dic/CMakeLists.txt deleted file mode 100644 index 726579b2..00000000 --- a/src/cor/lyt/dic/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_library(dic - OBJECT - dic.cpp - dic_fac.cpp -) - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -if (ENABLE_IWYU) - set_property(TARGET dic PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () diff --git a/src/cor/lyt/dic/dic.cpp b/src/cor/lyt/dic/dic.cpp deleted file mode 100644 index a8ee486a..00000000 --- a/src/cor/lyt/dic/dic.cpp +++ /dev/null @@ -1,275 +0,0 @@ -#include "fls/cor/lyt/dic/dic.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/eng/hzl_cmp_fac.hpp" -#include "fls/cor/eng/hzl_de_cmp_fac.hpp" -#include "fls/cor/prm/fsst/fsst.h" -#include "fls/logger/logger.hpp" -#include -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -DictHdrT DictHdrT::load(const uint8_t* p) { - DictHdrT dict_hdr; - std::memcpy(&dict_hdr, p, sizeof(dict_hdr)); - return dict_hdr; -} - -Dic::Dic(AnalyzeState& stt) - : data_buf {stt.data_buf} - , offs_buf {stt.offs_buf} - , compressed_data_buf {stt.compressed_data_buf} - , compressed_offs_buf {stt.compressed_offs_buf} { -} - -Dic::Dic(DecompressState& stt) - : data_buf {stt.data_buf} - , offs_buf {stt.offs_buf} - , compressed_data_buf {stt.decompressed_data_buf} - , compressed_offs_buf {stt.decompressed_offs_buf} { -} - -Dic::~Dic() = default; - -template -TDic::TDic(AnalyzeState& stt) - : Dic {stt} { - hzl_cmp_sp = HzlCmpFac::Create(stt, PageT::DICT_PAGE); - hzl_cmp_sp->Reset(); -}; - -template -TDic::TDic(DecompressState& stt) - : Dic {stt} { - hzl_de_cmp_sp = HzlDeCmpFac::Create(stt.exp_t, 0, PageT::DICT_PAGE); -}; - -template -n_t TDic::size() { - auto ttl_sz {sizeof(DictHdrT)}; - ttl_sz += compressed_offs_buf.length(); - - if constexpr (std::is_same() || std::is_same_v) { - FLS_ASSERT_ZERO(offs_buf.length()) - // always compress - ttl_sz += compressed_data_buf.length(); - } else if constexpr (std::is_same()) { - ttl_sz += compressed_data_buf.length(); - ttl_sz += FSST_MAXHEADER; - } else { - FLS_ABORT("NOT SUPPORTED") - } - - return ttl_sz; -} - -template -void TDic::Compress(CompressState& stt) { - stt.c = static_cast(map.size() + exc_map.size()); - - if constexpr (std::is_same_v) { - serialize(); - hzl_cmp_sp->FFOR(data_buf, compressed_data_buf, stt); - } else if constexpr (std::is_same_v) { - serialize(); - hzl_cmp_sp->ALP(data_buf, compressed_data_buf, stt); - } else if constexpr (std::is_same_v) { - serialize(); - hzl_cmp_sp->FSST(data_buf, offs_buf, compressed_data_buf, compressed_offs_buf, stt); - - } else { - FLS_ABORT("NOT SUPPORTED") - } -} - -template -void TDic::Write(Buf& buf, CompressState& stt) { - /* HDR*/ - DictHdrT dict_hdr = {ExpT::DICT, 0, 0, 0, static_cast(map.size() + exc_map.size())}; - - buf.UnsafeAppend(&dict_hdr, sizeof(dict_hdr)); - - if constexpr (std::is_same_v || std::is_same_v) { - FLS_ASSERT_ZERO(offs_buf.length()) - buf.UnsafeAppend(compressed_data_buf.data(), compressed_data_buf.length()); - - } else if constexpr (std::is_same_v) { - /* FSST */ - fsst_export(static_cast(stt.fsst_encoder), buf.end()); - buf.Advance(FSST_MAXHEADER); - - /* Offsets */ - buf.UnsafeAppend(compressed_offs_buf.data(), compressed_offs_buf.length()); - - /* Data */ - buf.UnsafeAppend(compressed_data_buf.data(), compressed_data_buf.length()); - } else { - FLS_ABORT("NOT SUPPORTED") - } -} - -template -void TDic::repetition_to_index() { - // [fixme] - FLS_PLOG_KEY_VALUE("fixme", "unordered_map has been used") - idx_t idx = 0; - for (auto& it : map) { - auto& c = it.second; - c = idx; - idx += 1; - } - - for (auto& it : exc_map) { - auto& c = it.second; - c = idx; - idx += 1; - } -} - -template -void TDic::Decompress(const uint8_t* a_src_p, DecompressState& stt) { - - auto* src_p = const_cast(a_src_p); - /* HDR*/ - DictHdrT dict_hdr; - std::memcpy(&dict_hdr, src_p, sizeof(dict_hdr)); - src_p += sizeof(dict_hdr); - - FLS_ASSERT_NOT_ZERO(dict_hdr.entry_c) - - if constexpr (std::is_same_v) { - hzl_de_cmp_sp->UNFFOR(const_cast(src_p), data_buf); - } - /**/ - else if constexpr (std::is_same_v) { - hzl_de_cmp_sp->UNFSST(const_cast(src_p), offs_buf, data_buf, dict_hdr.entry_c); - } - /**/ - else if constexpr (std::is_same_v) { - std::memcpy(data_buf.mutable_data(), src_p, dict_hdr.entry_c * sizeof(PT)); - // hzl_de_cmp_sp->UNALP(const_cast(src_p), data_buf); TODO - } - /**/ - else { - FLS_ABORT("NOT IMPLEMENTED") - throw std::runtime_error("NOT IMPLEMENTED!"); - } -} - -template -up TDic::get_idx_vec(const ofs_t* offset_arr, const void* a_byte_arr) { - auto idx_res_vec = std::make_unique>(); - auto last_idx = map.size() + exc_map.size(); - - if constexpr (std::is_same_v || std::is_same_v) { - const auto* in_arr = reinterpret_cast(a_byte_arr); - for (n_t i {0}; i < vec_n_tup(); ++i) { - auto val = in_arr[i]; - auto it = map.find(val); - if (it == map.end()) { - auto sec_it = exc_map.find(val); - if (sec_it == exc_map.end()) { - exc_map.emplace(val, last_idx); - idx_res_vec->push_back(last_idx); - last_idx++; - } else { - idx_res_vec->push_back(sec_it->second); - } - } else { - idx_res_vec->push_back(it->second); - } - } - return idx_res_vec; - } else if constexpr (std::is_same()) { - auto byte_arr = reinterpret_cast(a_byte_arr); - - idx_t exc_idx = map.size(); - for (ofs_t i {0}, start_off {0}, end_off {0}; i < vec_n_tup(); ++i, start_off = end_off) { - end_off = offset_arr[i]; - std::string next_string(&byte_arr[start_off], &byte_arr[end_off]); - auto it = map.find(next_string); - if (it == map.end()) { - idx_res_vec->push_back(exc_idx); - exc_idx++; - } else { - idx_res_vec->push_back(it->second); - } - } - return idx_res_vec; - } else { - FLS_ABORT("NOT SUPPORTED!") - return nullptr; - } -} - -template -bool sort_val_ascending(std::pair& a, std::pair& b) { - return a.second < b.second; -} - -template -void TDic::serialize() { - FLS_ASSERT_FALSE(map.empty()) - - if constexpr (std::is_same_v || std::is_same_v) { - auto* val_arr {reinterpret_cast(data_buf.mutable_data())}; - - // primary map - for (auto& it : map) { - auto& val = it.first; - auto& key = it.second; - val_arr[key] = val; - } - - // secondary map - for (auto& it : exc_map) { - auto& val = it.first; - auto& key = it.second; - val_arr[key] = val; - } - n_t ttl_sz = (map.size() + exc_map.size()) * sizeof(PT); - data_buf.Advance(ttl_sz); - } - /**/ - else if constexpr (std::is_same()) { - /* INDEX1_OFF_ARR*/ - uint32_t cur_ofs {0UL}; - - // [fixme] not optimal - // sort by index - std::vector> elems(map.begin(), map.end()); - std::sort(elems.begin(), elems.end(), sort_val_ascending); - - for (auto& it : elems) { - auto& str = it.first; - FLS_ASSERT_LE(cur_ofs, data_buf.Capacity()) // todo [buf] - - offs_buf.UnsafeAppend(&cur_ofs, sizeof(ofs_t)); - data_buf.UnsafeAppend(str.data(), str.size()); - - cur_ofs += str.size(); - } - - offs_buf.UnsafeAppend(&cur_ofs, sizeof(ofs_t)); - - // [fixme] add exc_map - } - /**/ - else { - FLS_ABORT("NOT SUPPORTED!") - } -} - -template -TDic::~TDic() = default; - -/*--------------------------------------------------------------------------------------------------------------------*\ -+ Specialization -\*--------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(TDic) - -} // namespace fastlanes diff --git a/src/cor/lyt/dic/dic_fac.cpp b/src/cor/lyt/dic/dic_fac.cpp deleted file mode 100644 index 984a15ab..00000000 --- a/src/cor/lyt/dic/dic_fac.cpp +++ /dev/null @@ -1,581 +0,0 @@ -#include "fls/cor/lyt/dic/dic_fac.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/lyt/dic/dic.hpp" -#include "fls/utl/random.hpp" -#include "fls/utl/util.hpp" -#include -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -enum class DictSortStrategy : uint8_t { // nolint - SORT_BY_INSERTION_ORDER = 0, - SORT_BY_FREQUENCY = 1, - SORT_BY_VALUE = 2, -}; - -constexpr DictSortStrategy get_strategy(ExpT exp_t) { - switch (exp_t) { - case ExpT::INVALID: - case ExpT::UNDECIDED: - case ExpT::UNCOMPRESSED: - case ExpT::BYTE_ARR: - case ExpT::FFOR: - case ExpT::DELTA: - case ExpT::RLE: - case ExpT::FSST: - case ExpT::DICT: - return DictSortStrategy::SORT_BY_INSERTION_ORDER; - case ExpT::DICT_RLE: - return DictSortStrategy::SORT_BY_INSERTION_ORDER; - case ExpT::DICT_FRQ: - return DictSortStrategy::SORT_BY_FREQUENCY; - case ExpT::DICT_VAL: - return DictSortStrategy::SORT_BY_VALUE; - // TODO [FIXME] - default: - return DictSortStrategy::SORT_BY_INSERTION_ORDER; - } -} - -/* Flip a map*/ -template -static std::multimap invert(std::unordered_map& map) { - if constexpr (std::is_same()) { - std::multimap multi_map; - - std::unordered_map::iterator it; - for (it = map.begin(); it != map.end(); it++) { - multi_map.insert(std::make_pair(it->second, it->first)); - } - - return multi_map; - } else if constexpr (std::is_same()) { - std::multimap multi_map; - - std::unordered_map::iterator it; - for (it = map.begin(); it != map.end(); it++) { - multi_map.insert(std::make_pair(it->second, it->first)); - } - - return multi_map; - } -} - -template -up> dict_fac::adaptive_create(const PT* data_p, n_t n, AnalyzeState& stt) { - FLS_ASSERT_CORRECT_N(n) - - const n_t vec_c {n / CFG::VEC_TUP_C}; - n_t analyzed_vec_c {0}; - if (vec_c == 0) { - return nullptr; - }; /* No dict for tuple-c < 1024 */ - - /* Init. */ - //[fixme] move to stt - vector is_processed; - is_processed.reserve(vec_c); - - n_t result {0}; - - /* base. */ - auto base_dict = dict_fac::m_create_from(data_p, 0, stt); - analyzed_vec_c++; - if (vec_c == 1) { - base_dict->repetition_to_index(); - return base_dict; - } - - /* next. */ - auto next_dict = dict_fac::m_create_random(data_p, vec_c, is_processed, stt); - analyzed_vec_c++; - - result = dict_fac::m_compare(base_dict->map, next_dict->map); - dict_fac::m_combine(base_dict->map, next_dict->map); - - /* Adaptive analyzing*/ - while (result < CFG::DIC::THRESHOLD) { - FLS_ASSERT_LE(analyzed_vec_c, vec_c) - - if (analyzed_vec_c == vec_c) { - break; - } - next_dict = dict_fac::m_create_random(data_p, vec_c, is_processed, stt); - - /* Check the dic_page limit. */ - if (base_dict->size() + next_dict->size() > CFG::DIC::DIC_PAGE_SZ) { - break; - } - - result = dict_fac::m_compare(base_dict->map, next_dict->map); - dict_fac::m_combine(base_dict->map, next_dict->map); - } - - base_dict->repetition_to_index(); - return base_dict; -} - -template -up> -dict_fac::m_create_random(const PT* data_p, n_t possible_sample_c, vector& is_processed, AnalyzeState& stt) { - auto next_batch_num = random::next_vec_num(possible_sample_c, is_processed); - return m_create_from(data_p, next_batch_num, stt); -} - -template -n_t dict_fac::m_compare(dic_t& r, dic_t& l) { - if constexpr (std::is_same()) { - n_t total {0}; - - std::unordered_map::iterator r_it; - - /* How many tuples are covered by r dic?*/ - for (r_it = r.begin(); r_it != r.end(); r_it++) { - auto l_it = l.find(r_it->first); - - if (l_it != l.end()) { - total += l_it->second; - } - } - - return total; - } else if constexpr (std::is_same()) { - n_t total {0}; - - std::unordered_map::iterator r_it; - - /* How many tuples are covered by r dic?*/ - for (r_it = r.begin(); r_it != r.end(); r_it++) { - auto l_it = l.find(r_it->first); - - if (l_it != l.end()) { - total += l_it->second; - } - } - - return total; - } else { - FLS_ABORT("NOT SUPPORTED!") - return 0; - } -} - -template -up> dict_fac::m_create_from(const PT* data_p, n_t sample_num, AnalyzeState& stt) { - if constexpr (std::is_same()) { - auto dic = std::make_unique>(stt); - - /* Create Dic */ - for (size_t i = 0; i < CFG::VEC_TUP_C; ++i) { - auto next_val = data_p[i]; - if (dic->map.find(next_val) == dic->map.end()) { - dic->map.emplace(next_val, 1); - } else { - dic->map[next_val]++; - } - } - - /* Prune the 10 percent undesired part. */ - std::multimap flipped_map = invert(dic->map); - n_t sum {0}; - - auto it = flipped_map.end(); - it--; - for (; it != flipped_map.begin();) { - sum += it->first; - --it; - if (sum >= CFG::DIC::THRESHOLD) { - /* Remove elements from now on. */ - dic->map.erase(dic->map.find(it->second)); - } - } - - return dic; - - } else if constexpr (std::is_same()) { - auto dic_up = std::make_unique>(stt); - - /* Create Dic */ - for (size_t i = 0; i < CFG::VEC_TUP_C; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - } - - /* Prune the 10 percent undesired part. */ - std::multimap flipped_map = invert(dic_up->map); - n_t visited {0}; - - for (auto it = flipped_map.end(); it != flipped_map.begin(); --it) { - visited += it->first; - if (visited >= CFG::DIC::THRESHOLD) { - /* Remove element from now on. */ - dic_up->map.erase(dic_up->map.find(it->second)); - } - } - - return dic_up; - } else { - FLS_ABORT("NOT SUPPORTED!") - return nullptr; - } -} - -template -void dict_fac::m_combine(dic_t& left, dic_t& right) { - if constexpr (std::is_same()) { - for (auto& r_it : right) { - auto l_it = left.find(r_it.first); - if (l_it != left.end()) { - left.emplace(r_it.first, r_it.second); - } else { - left[r_it.first] += r_it.second; - } - } - } - - else { - FLS_ABORT("NOT SUPPORTED!") - } -} - -template -bool sort_val_ascending(std::pair& a, std::pair& b) { - return a.second < b.second; -} - -template -bool sort_val_descending(std::pair& a, std::pair& b) { - return a.second > b.second; -} - -template -bool sort_key_ascending(std::pair& a, std::pair& b) { - return a.first < b.first; -} - -template -bool sort_key_descending(std::pair& a, std::pair& b) { - return a.first > b.first; -} - -template -up> dict_fac::perfect_create(const PT* data_p, n_t n, AnalyzeState& stt) { - if constexpr (std::is_same_v || std::is_same_v) { - auto dic_up = std::make_unique>(stt); - std::vector> elems; - auto strategy = get_strategy(stt.m_exp_t); - - switch (strategy) { - case DictSortStrategy::SORT_BY_INSERTION_ORDER: { - /* Create Dic */ - auto idx = 0; - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, idx++); - } - }; - - // pair of value - idx - elems = {dic_up->map.begin(), dic_up->map.end()}; - - // sort based on frequency! - std::sort(elems.begin(), elems.end(), sort_val_ascending); - } break; - case DictSortStrategy::SORT_BY_FREQUENCY: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - }; - - // pair of value - frequency - elems = {dic_up->map.begin(), dic_up->map.end()}; - - // sort based on frequency! - std::sort(elems.begin(), elems.end(), sort_val_descending); - - } break; - case DictSortStrategy::SORT_BY_VALUE: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - }; - - // pair of value - frequency - elems = {dic_up->map.begin(), dic_up->map.end()}; - - // sort based on frequency! - std::sort(elems.begin(), elems.end(), sort_key_ascending); - } - } - - dic_up->map.clear(); - - n_t idx {0}; - for (auto& pair : elems) { - dic_up->map.emplace(pair.first, idx); - idx++; - } - - return dic_up; - } else if constexpr (std::is_same()) { - auto dic_up = std::make_unique>(stt); - auto strategy = get_strategy(stt.m_exp_t); - std::vector> elems; - - switch (strategy) { - case DictSortStrategy::SORT_BY_INSERTION_ORDER: { - /* Create Dic */ - auto idx = 0; - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, idx++); - } - }; - - elems = {dic_up->map.begin(), dic_up->map.end()}; - std::sort(elems.begin(), elems.end(), sort_val_descending); - } break; - case DictSortStrategy::SORT_BY_FREQUENCY: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - } - - elems = {dic_up->map.begin(), dic_up->map.end()}; - std::sort(elems.begin(), elems.end(), sort_val_descending); - - } break; - case DictSortStrategy::SORT_BY_VALUE: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - }; - - elems = {dic_up->map.begin(), dic_up->map.end()}; - - std::sort(elems.begin(), elems.end(), sort_key_ascending); - } - } - - dic_up->map.clear(); - - n_t idx {0}; - for (auto& pair : elems) { - dic_up->map.emplace(pair.first, idx); - idx++; - } - - return dic_up; - } else { - FLS_ABORT("NOT SUPPORTED!") - return nullptr; - } -} - -template -bsz_t dict_fac::multi_dic_create(const std::vector& src_col, - AnalyzeState& src_stt, - const std::vector& tgt_col, - AnalyzeState& tgt_stt) { - bsz_t ttl_sz {0}; - - if constexpr (std::is_same_v) { - auto src_dic = *dynamic_cast*>(src_stt.dic_up.get()); - auto tgt_dic = *dynamic_cast*>(tgt_stt.dic_up.get()); - - FLS_ABORT("HERE") - - } else if constexpr (std::is_same_v) { - } else { - } - - return ttl_sz; -} - -template -bsz_t dict_fac::multi_dic_create(const std::vector& src_col, - AnalyzeState& src_stt, - const std::vector& tgt_col, - AnalyzeState& tgt_stt) { - bsz_t ttl_sz {0}; - - FLS_ABORT("HERE") - if constexpr (std::is_same_v) { - - } else if constexpr (std::is_same_v) { - } else { - } - - return ttl_sz; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col, const std::vector& tgt_col) { - FLS_ASSERT_EQUALITY(src_col.size(), tgt_col.size()) - - bsz_t ttl_size {0}; - - std::unordered_map> new_dic; - - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - const auto& tgt_val = tgt_col[idx]; - - if (new_dic.find(src_val) == new_dic.end()) { - new_dic.insert({src_val, {}}); - } else { - new_dic[src_val].insert(tgt_val); - } - } - - sz_t max_sz {0}; - - for (const auto& map : new_dic) { - auto map_size = map.second.size(); - if (max_sz < map.second.size()) { - max_sz = map_size; - } - - for (const auto& val : map.second) { - ttl_size += val.size() + sizeof(ofs_t); - } - } - - auto bits_per_value = RANGE_BIT(max_sz); - auto ttl_bytes = src_col.size() * bits_per_value / 8; - return ttl_size + ttl_bytes; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col) { - - bsz_t ttl_size {0}; - - std::set dic; - - if constexpr (!std::is_same_v) { - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - if (dic.find(src_val) == dic.end()) { - dic.insert(src_val); - } - } - } else { - FLS_ABORT("NOT IMPLEMENTED TYPE") - } - - if constexpr (std::is_same_v) { - ttl_size += dic.size() * (sizeof(PT) + sizeof(ofs_t)); - } else if constexpr (std::is_same_v) { - for (const auto& val : dic) { - ttl_size += val.size() + sizeof(ofs_t); - } - } else { - FLS_ABORT("NOT IMPLEMENTED TYPE") - } - - return ttl_size; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col, const std::vector& tgt_col) { - FLS_ASSERT_EQUALITY(src_col.size(), tgt_col.size()) - - bsz_t ttl_size {0}; - - std::unordered_map> new_dic; - - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - const auto& tgt_val = tgt_col[idx]; - - if (new_dic.find(src_val) == new_dic.end()) { - new_dic.insert({src_val, {}}); - } else { - new_dic[src_val].insert(tgt_val); - } - } - - sz_t max_sz {0}; - for (const auto& map : new_dic) { - auto map_size = map.second.size(); - if (max_sz < map.second.size()) { - max_sz = map_size; - } - - ttl_size += map.second.size() * (sizeof(PT) + sizeof(ofs_t)); - } - - auto bits_per_value = RANGE_BIT(max_sz); - auto ttl_bytes = (src_col.size() * bits_per_value / 8); - return ttl_size + ttl_bytes; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col, const std::vector& tgt_col) { - FLS_ASSERT_EQUALITY(src_col.size(), tgt_col.size()) - - bsz_t ttl_size {0}; - - std::unordered_map> new_dic; - - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - const auto& tgt_val = tgt_col[idx]; - - if (new_dic.find(src_val) == new_dic.end()) { - new_dic.insert({src_val, {}}); - } else { - new_dic[src_val].insert(tgt_val); - } - } - - sz_t max_sz {0}; - for (const auto& map : new_dic) { - auto map_size = map.second.size(); - if (max_sz < map.second.size()) { - max_sz = map_size; - } - - ttl_size += map.second.size() * (sizeof(PT) + sizeof(ofs_t)); - } - - auto bits_per_value = RANGE_BIT(max_sz); - auto ttl_bytes = (src_col.size() * bits_per_value / 8); - return ttl_size + ttl_bytes; -} - -/*--------------------------------------------------------------------------------------------------------------------*\ -+ Specialization -\*--------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(dict_fac) -} // namespace fastlanes diff --git a/src/cor/lyt/page/CMakeLists.txt b/src/cor/lyt/page/CMakeLists.txt deleted file mode 100644 index bbf75bbd..00000000 --- a/src/cor/lyt/page/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_library(page - OBJECT - page.cpp - page_hdr.cpp - page_param.cpp - entry_point.cpp - type.cpp - offset.cpp -) - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -if (ENABLE_IWYU) - set_property(TARGET page PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () diff --git a/src/cor/lyt/page/entry_point.cpp b/src/cor/lyt/page/entry_point.cpp deleted file mode 100644 index 66b0b635..00000000 --- a/src/cor/lyt/page/entry_point.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "fls/cor/lyt/page/entry_point.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include -#include - -namespace fastlanes { -void EpArr::InitDecompress(Page& page) { - FLS_ASSERT_NOT_NULL_POINTER(page.start_p) - FLS_ASSERT_CORRECT_N(page.vec_c()) - FLS_ASSERT_CORRECT_N(page.arr_c) - - m_vec_c = page.vec_c(); - m_arr_c = page.arr_c; - m_vec_num = 0; - m_start_p = page.start_p + page.off_arr.ttl_sz() + CFG::PG::HDR_SZ; -} - -void EpArr::load_and_next() { - FLS_ASSERT_CORRECT_NUM(m_vec_num, m_vec_c) - FLS_ASSERT_NOT_NULL_POINTER(m_start_p) - FLS_ASSERT_CORRECT_N(m_vec_c) - - FLS_PLOG_KEY_VALUE("vec_n", std::to_string(m_vec_num)) - FLS_PLOG_KEY_VALUE("vec_c", std::to_string(m_vec_c)) - - /* the first ep_arr is always 0 and has not been stored! */ - if (m_vec_num == 0) { - memset(arr, 0, m_arr_c * CFG::PG::ENTRY_POINT_SZ); - m_vec_num++; - FLS_PLOG_KEY_VALUE("ep", ToStr::to_str(*this)) - return; - } - - memcpy(arr, m_start_p, m_arr_c * CFG::PG::ENTRY_POINT_SZ); - FLS_PLOG_KEY_VALUE("ep", ToStr::to_str(*this)) - m_start_p += m_arr_c * CFG::PG::ENTRY_POINT_SZ; - m_vec_num++; -} - -void EpArr::InitCompress(Page& page) { - FLS_ASSERT_CORRECT_N(page.arr_c) - - m_arr_c = page.arr_c; - m_vec_c = 0; - m_vec_num = 0; - - /* init all entry points to 0. */ - for (auto& ep : arr) { - ep = 0; - } -} - -n_t EpArr::sz() { - return m_arr_c * CFG::PG::ENTRY_POINT_SZ; -} -void EpArr::Reset() { - m_vec_num = 0; - m_vec_c = 0; -} - -EpArr::EpArr() = default; -} // namespace fastlanes diff --git a/src/cor/lyt/page/offset.cpp b/src/cor/lyt/page/offset.cpp deleted file mode 100644 index 6588712f..00000000 --- a/src/cor/lyt/page/offset.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "fls/cor/lyt/page/offset.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include -#include - -namespace fastlanes { -OffArr::OffArr() = default; - -void OffArr::InitDecompress(Page& page) { - FLS_ASSERT_CORRECT_N(page.arr_c) - FLS_ASSERT_NOT_NULL_POINTER(page.start_p) - - m_arr_c = page.arr_c; - memcpy(arr, page.start_p + CFG::PG::HDR_SZ, m_arr_c * CFG::PG::OFFSET_SZ); - FLS_PLOG_KEY_VALUE("off_arr", ToStr::to_str(*this).c_str()) -} - -void OffArr::InitCompress(n_t arr_c) { - FLS_ASSERT_CORRECT_N(arr_c) - m_arr_c = arr_c; -} - -n_t OffArr::ttl_sz() { - FLS_ASSERT_CORRECT_SZ(m_arr_c) - return m_arr_c * CFG::PG::OFFSET_SZ; -} -n_t OffArr::arr_c() { - return m_arr_c; -} -} // namespace fastlanes diff --git a/src/cor/lyt/page/page.cpp b/src/cor/lyt/page/page.cpp deleted file mode 100644 index f9a75875..00000000 --- a/src/cor/lyt/page/page.cpp +++ /dev/null @@ -1,181 +0,0 @@ -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/eng/hzl_cmp.hpp" -#include "fls/cor/exp/exp.hpp" -#include "fls/cor/exp/rpn.hpp" -#include "fls/cor/lyt/rec_hdr.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -Page::Page() - : m_ttl_sz {0} { -} - -void Page::Reset() { - for (auto& buf : buf_arr) { - buf.Reset(); - } - tup_c = 0; - arr_c = 0; - m_ttl_sz = 0; - pg_hdr.Reset(); - ep_arr.Reset(); -} - -void Page::InitCompress(const sp& exp_sp) { - for (auto& buff : buf_arr) { - FLS_UNUSED(buff.Rewind(0)); - } - ep_arr_buf.Rewind(0); - - /* Page hdr.*/ - pg_hdr = exp_sp->page_hdr(); - - /* Arr offsets: */ - arr_c = exp_sp->arw_c(); - off_arr.InitCompress(arr_c); - - /* Entry Point: */ - ep_arr.InitCompress(*this); - - /* Current size.*/ - m_ttl_sz = 0; - m_ttl_sz += CFG::PG::HDR_SZ; - m_ttl_sz += arr_c * CFG::PG::OFFSET_SZ; - - /* Add rec buff hdrs*/ - auto* rpn = exp_sp->cmpr_rpn(); - for (n_t i {0}; i < rpn->des_buf_c; ++i) { - if (!rpn->is_compressed[i]) { - continue; - } - - m_ttl_sz += sizeof(rec_hdr); - } - - /* Start pointer*/ - start_p = nullptr; - - FLS_PLOG_KEY_VALUE("initial_sz", std::to_string(m_ttl_sz)) -} - -void Page::InitDecompress(const uint8_t* p, const sp& exp_up) { - FLS_ASSERT_NOT_NULL_POINTER(p) - - /* Start pointer*/ - start_p = const_cast(p); - - /* Page hdr.*/ - pg_hdr.Load(p); - - /* Arr offsets: */ - arr_c = exp_up->arw_c(); - off_arr.InitDecompress(*this); - - /* Init Params: */ - params.Init(*this); - - /* Init EP: */ - ep_arr.InitDecompress(*this); - - tup_c = 0; - - FLS_PLOG_KEY_VALUE("pg", ToStr::to_str(*this)); -} - -PageHdr Page::hdr() { - return pg_hdr; -} - -void Page::Absorb(Vec& vec) { - if (vec_c() >= 1) { - ep_arr_buf.UnsafeAppend(ep_arr.arr, ep_arr.sz()); - m_ttl_sz += ep_arr.sz(); - } - - n_t vec_ttl_sz {0}; - for (size_t i {0}; i < arr_c; ++i) { - auto added_sz = vec.buf_arr[i].length(); - buf_arr[i].UnsafeAppend(vec.buf_arr[i].data(), added_sz); - ep_arr.arr[i] = ep_arr.arr[i] + added_sz; - vec_ttl_sz += added_sz; - } - - m_ttl_sz += vec_ttl_sz; - pg_hdr.vec_c++; - - FLS_PLOG_KEY_VALUE("ep", ToStr::to_str(ep_arr)) - FLS_PLOG_KEY_VALUE("ep_arr", - ToStr::to_str(reinterpret_cast(ep_arr_buf.mutable_data()), pg_hdr.vec_c)) - FLS_PLOG_KEY_VALUE("ttl_sz", std::to_string(m_ttl_sz)); - FLS_PLOG_KEY_VALUE("vec_ttl_sz", std::to_string(vec_ttl_sz)); - FLS_PLOG_KEY_VALUE("vec_c", std::to_string(vec_c())); - - vec.Reset(); -} - -void Page::vec_c_inc() { - pg_hdr.vec_c++; -} - -uint16_t Page::vec_c() { - return pg_hdr.vec_c; -} - -void Page::OffsetToSink(Buf& sink) { - /* First one - * Rest follow each other.*/ - off_arr.arr[0] = CFG::PG::HDR_SZ + off_arr.ttl_sz() + ep_arr_buf.length(); - for (size_t i {1}; i < arr_c; ++i) { - off_arr.arr[i] = off_arr.arr[i - 1] + buf_arr[i - 1].length(); - } - - sink.UnsafeAppend(off_arr.arr, off_arr.ttl_sz()); - FLS_PLOG_KEY_VALUE("OFF_arr", ToStr::to_str(off_arr)) -} - -PageParam Page::page_params() { - PageParam res; - ep_arr.load_and_next(); - for (size_t i {0}; i < arr_c; ++i) { - res.arr[i] = params.arr[i] + ep_arr.arr[i]; - } - - return res; -} - -n_t Page::size_of(Vec& vec) { - n_t vec_ttl_sz {0}; - - if (vec_c() >= 1) { - vec_ttl_sz = ep_arr.sz(); - } - - for (size_t i {0}; i < arr_c; ++i) { - auto added_sz = vec.buf_arr[i].length(); - vec_ttl_sz += added_sz; - } - - return vec_ttl_sz; -} - -n_t Page::get_total_sz() { - n_t ttl_sz {0}; - for (auto& buf : buf_arr) { - ttl_sz += buf.length(); - } - - return ttl_sz; -} - -bool Page::Verify() { - auto ttl_sz = get_total_sz(); - return ttl_sz <= CFG::PG::SZ; -} - -Page::~Page() = default; -} // namespace fastlanes diff --git a/src/cor/lyt/page/page_hdr.cpp b/src/cor/lyt/page/page_hdr.cpp deleted file mode 100644 index cf2a7a86..00000000 --- a/src/cor/lyt/page/page_hdr.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "fls/cor/lyt/page/page_hdr.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/common/assert.hpp" -#include - -namespace fastlanes { -void PageHdr::Load(const uint8_t* p) { - FLS_ASSERT_NOT_NULL_POINTER(p) - - *this = *reinterpret_cast(p); -} - -uint32_t PageHdr::Serialize() { - return 0; -} - -void PageHdr::AddCompressedArrBitMap() { -} -void PageHdr::Reset() { - vec_c = 0; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Static Assert: -\---------------------------------------------------------------------------------------------------------------------*/ -static_assert(std::is_standard_layout() && std::is_trivial(), - "PageHdr should be a standard-layout trivial type."); -static_assert(sizeof(PageHdr) == CFG::PG::HDR_SZ, "PageHdr should be 4-byte long."); -} // namespace fastlanes diff --git a/src/cor/lyt/page/page_param.cpp b/src/cor/lyt/page/page_param.cpp deleted file mode 100644 index 0990c483..00000000 --- a/src/cor/lyt/page/page_param.cpp +++ /dev/null @@ -1,13 +0,0 @@ -#include "fls/cor/lyt/page/page_param.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include - -namespace fastlanes { -void PageParam::Init(Page& page) { - /* Pointer = start_p + offset! */ - for (size_t i {0}; i < page.arr_c; ++i) { - arr[i] = page.start_p + page.off_arr.arr[i]; - } -} -} // namespace fastlanes diff --git a/src/cor/lyt/page/type.cpp b/src/cor/lyt/page/type.cpp deleted file mode 100644 index cd8f4deb..00000000 --- a/src/cor/lyt/page/type.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "fls/cor/lyt/page/type.hpp" -#include "fls/common/common.hpp" -#include - -namespace fastlanes { -std::string to_string(PageT t) { - switch (t) { - case PageT::COMPRESS: - return {"COMPRESS"}; - case PageT::DECOMPRESS: - return {"DECOMPRESS"}; - case PageT::INVALID: - default: - return {"INVALID"}; - } - - return {}; -} -} // namespace fastlanes diff --git a/src/cor/lyt/vec.cpp b/src/cor/lyt/vec.cpp deleted file mode 100644 index 9c9ca594..00000000 --- a/src/cor/lyt/vec.cpp +++ /dev/null @@ -1,365 +0,0 @@ -#include "fls/cor/lyt/vec.hpp" -#include "fls/common/string.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/eng/engine.hpp" -#include "fls/cor/lyt/dic/dic.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/prm/fsst/fsst_prm.hpp" -#include "fls/cor/prm/fsst12/fsst12_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include - -#pragma clang diagnostic ignored "-Wshorten-64-to-32" - -namespace fastlanes { -static constexpr uint64_t VEC_SZ = 1024ULL * 1024; - -Vec::Vec() - : buf_arr {Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ)} - , arr_c {0} - , tup_c {0} { -} - -void Vec::Reset() { - for (auto& buff : buf_arr) { - buff.Rewind(0); - } - - arr_c = 0; -} - -void Vec::FlushToPage(Page& page) { - for (size_t i {0}; i < arr_c; ++i) { - n_t sz = buf_arr[i].length(); - page.buf_arr[i].UnsafeAppend(buf_arr[i].data(), buf_arr[i].length()); - page.ep_arr_buf.UnsafeAppend(&sz, sizeof(uint16_t)); - } -} - -template -void Vec::Init(const T* in_arr, EngineState& stt) { - if constexpr (std::is_arithmetic_v) { - FLS_ASSERT_NOT_NULL_POINTER(in_arr) - - auto exp_t = stt.analyze_state_up->m_exp_t; - FLS_ASSERT_CORRECT_EXP_T(static_cast(exp_t)) - - if constexpr (std::is_same_v || std::is_same_v) { - Reset(); - switch (exp_t) { - case ExpT::RLE: { - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - auto cur_val = in_arr[i]; - - if (i == 0) { - cur_val = in_arr[0]; - pos_val = 0; - buf_arr[0].UnsafeAppend(&cur_val, sizeof(T)); - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - continue; - } - - if (cur_val != in_arr[i - 1]) { - buf_arr[0].UnsafeAppend(&cur_val, sizeof(T)); - pos_val += 1; - } - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - } - - return; - } - case ExpT::DICT_RLE: { - auto idx_vec = - stt.analyze_state_up->dic_up->get_idx_vec(nullptr, reinterpret_cast(in_arr)); - - auto* idx_arr = reinterpret_cast(idx_vec->data()); - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - int64_t cur_val = idx_arr[i]; - - if (i == 0) { - cur_val = idx_arr[0]; - pos_val = 0; - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - continue; - } - - if (cur_val != idx_arr[i - 1]) { - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - pos_val += 1; - } - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - } - arr_c += 1; - } - case ExpT::DICT_VAL: - case ExpT::DICT_FRQ: - case ExpT::DICT: { - auto idx_arr = stt.analyze_state_up->dic_up->get_idx_vec(nullptr, in_arr); - - buf_arr[arr_c + 0].UnsafeAppend(idx_arr->data(), vec_bsz()); - - arr_c += 1; - return; - } - case ExpT::UNCOMPRESSED: - case ExpT::BYTE_ARR: - case ExpT::FFOR: - case ExpT::ALP: - case ExpT::DELTA: { - [[maybe_unused]] auto* val_arr = reinterpret_cast(buf_arr[0].mutable_data()); - buf_arr[arr_c + 0].UnsafeAppend(in_arr, vec_bsz()); - - arr_c += 1; - FLS_PLOG_KEY_VALUE("val_arr", ToStr::to_str(val_arr)); - return; - } - case ExpT::TGT_EQUALITY: - case ExpT::TGT_1T1: - case ExpT::TGT_1TN: - case ExpT::SRC_1T1: - case ExpT::SRC_1TN: - case ExpT::ALP_RD: - break; - case ExpT::INVALID: - case ExpT::UNDECIDED: - case ExpT::FSST: - default: { - FLS_ABORT("EXP IS NOT SUPPORTED!") - return; - } - } - } - } - /**/ - else if (std::is_same_v) { - this->Reset(); - - auto exp_t = stt.analyze_state_up->m_exp_t; - FLS_ASSERT_CORRECT_EXP_T(static_cast(exp_t)) - - /*two buffs: hacky */ - Buf bytes; - Buf smart_offsets; - - /*Extract bytes*/ - for (idx_t idx {0}; idx < vec_n_tup(); ++idx) { - auto str = in_arr[idx]; - bytes.UnsafeAppend(str.data(), str.size()); - } - - /* Extract Offsets. - * Smart Offsets - */ - ofs_t cur_ofs {0}; - for (auto i {0ULL}; i < vec_n_tup(); ++i) { - cur_ofs += static_cast(in_arr[i].size()); - smart_offsets.UnsafeAppend(&cur_ofs, sizeof(ofs_t)); - } - - switch (exp_t) { - case ExpT::FSST: { - auto* encoder = reinterpret_cast(stt.analyze_state_up->fsst_encoder); - auto* str_arr = bytes.mutable_data(); - auto* off_arr = smart_offsets.mutable_data(); - auto str_p_in = Str::smart_offset_to_pointer(vec_sz(), off_arr, str_arr); - auto str_len_in = fsst_wrapper::offset_to_length(vec_sz(), off_arr); - - auto* out_p = buf_arr[arr_c + 0].mutable_data(); - auto* len_out = reinterpret_cast(buf_arr[arr_c + 3].mutable_data()); - auto* off_out = reinterpret_cast(buf_arr[arr_c + 2].mutable_data()); - /* - fsst_encoder_t *encoder, IN: encoder obtained from fsst_create(). - size_t nstrings, IN: number of strings in batch to compress. - size_t len_in[], IN: byte-lengths of the inputs - unsigned char *str_in[], IN: input string start pointers. - size_t outsize, IN: byte-length of output buffer. - unsigned char *output, OUT: memory buffer to put the compressed strings in (one after the other). - size_t len_out[], OUT: byte-lengths of the compressed strings. - unsigned char *str_out[] OUT: output string start pointers. Will all point into [output,output+size) - .*/ - - fsst_compress(encoder, // - vec_n_tup(), - str_len_in->data(), - str_p_in->data(), - smart_offsets.Capacity(), - out_p, - len_out, - off_out); - - // smart-offsets. - for (n_t i = 0; i < vec_n_tup() - 1; ++i) { - len_out[i] = static_cast(off_out[i + 1] - out_p); - FLS_ASSERT_CORRECT_SMART_OFFSET(off_out[i + 1] - out_p) - } - len_out[1023] = len_out[1022] + len_out[1023]; - - buf_arr[arr_c + 1].UnsafeAppend(len_out, vec_bsz()); - - arr_c += 2; - - FLS_PLOG_KEY_VALUE("byte_arr", ToStr::to_str(buf_arr[0].mutable_data())); - FLS_PLOG_KEY_VALUE("len_arr", ToStr::to_str(buf_arr[1].mutable_data())); - FLS_PLOG_KEY_VALUE("str_pointer", ToStr::to_str(buf_arr[2].mutable_data())); - } break; - case ExpT::FSST12: { - auto* encoder = reinterpret_cast(stt.analyze_state_up->fsst12_encoder); - auto* str_arr = bytes.mutable_data(); - auto* off_arr = smart_offsets.mutable_data(); - auto str_p_in = Str::smart_offset_to_pointer(vec_sz(), off_arr, str_arr); - auto str_len_in = fsst_wrapper::offset_to_length(vec_sz(), off_arr); - - auto* out_p = buf_arr[arr_c + 0].mutable_data(); - auto* len_out = reinterpret_cast(buf_arr[arr_c + 3].mutable_data()); - auto* off_out = reinterpret_cast(buf_arr[arr_c + 2].mutable_data()); - - /* Compress a batch of strings (on AVX512 machines best performance is obtained by compressing more than - * 32KB of string volume). */ - /* The output buffer must be large; at least "conservative space" (7+2*inputlength) for the first string for - * something to happen. */ - /* OUT: the number of compressed strings (<=n) that fit the output buffer. */ - /* IN: encoder obtained from fsst12_create(). */ - /* IN: number of strings in batch to compress. */ - /* IN: byte-lengths of the inputs */ - /* IN: input string start pointers. */ - /* IN: byte-length of output buffer. */ - /* OUT: memory buffer to put the compressed strings in (one after the other). */ - /* OUT: byte-lengths of the compressed strings. */ - /* OUT: output string start pointers. Will all point into [output,output+size). */ - - fsst12_compress(encoder, // - vec_n_tup(), - str_len_in->data(), - str_p_in->data(), - smart_offsets.Capacity(), - out_p, - len_out, - off_out); - - // smart-offsets. - for (n_t i = 0; i < vec_n_tup() - 1; ++i) { - len_out[i] = static_cast(off_out[i + 1] - out_p); - FLS_ASSERT_CORRECT_SMART_OFFSET(off_out[i + 1] - out_p) - } - len_out[1023] = len_out[1022] + len_out[1023]; - - buf_arr[arr_c + 1].UnsafeAppend(len_out, vec_bsz()); - - arr_c += 2; - - FLS_PLOG_KEY_VALUE("byte_arr", ToStr::to_str(buf_arr[0].mutable_data())); - FLS_PLOG_KEY_VALUE("len_arr", ToStr::to_str(buf_arr[1].mutable_data())); - FLS_PLOG_KEY_VALUE("str_pointer", ToStr::to_str(buf_arr[2].mutable_data())); - } break; - case ExpT::DICT_VAL: - case ExpT::DICT_FRQ: - case ExpT::DICT: { - auto idx_arr = stt.analyze_state_up->dic_up->get_idx_vec(smart_offsets.data(), bytes.data()); - - buf_arr[arr_c + 0].UnsafeAppend(idx_arr->data(), vec_bsz()); - - arr_c += 1; - - FLS_PLOG_KEY_VALUE("idx_arr", - ToStr::to_str(reinterpret_cast(buf_arr[0].mutable_data()))); - } break; - case ExpT::DICT_RLE: { - auto idx_vec = stt.analyze_state_up->dic_up->get_idx_vec(smart_offsets.data(), bytes.data()); - - auto* idx_arr = reinterpret_cast(idx_vec->data()); - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - int64_t cur_val = idx_arr[i]; - - if (i == 0) { - cur_val = idx_arr[0]; - pos_val = 0; - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - continue; - } - - if (cur_val != idx_arr[i - 1]) { - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - pos_val += 1; - } - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - } - arr_c += 2; - } break; - case ExpT::BYTE_ARR: { - buf_arr[arr_c + 0].UnsafeAppend(bytes.data(), bytes.length()); - buf_arr[arr_c + 1].Clone(smart_offsets); - arr_c += 2; - - FLS_PLOG_KEY_VALUE("byte_arr", ToStr::to_str(buf_arr[0].mutable_data())); - FLS_PLOG_KEY_VALUE("offset_arr", ToStr::to_str(buf_arr[1].mutable_data())); - } break; - default: - FLS_ABORT("EXP IS NOT SUPPORTED!") - } - } - /**/ - else { - FLS_ABORT("NOT SUPPORTED TYPE") - } -} - -void Vec::Log() { - // FLS_PRINT_4MSG("tup_c: ", std::to_string(tup_c), "arr_c", std::to_string(arr_c)) -} - -VecParam Vec::vec_param() { - return {buf_arr[0].mutable_data(), buf_arr[1].mutable_data(), buf_arr[2].mutable_data()}; -} - -template -void Vec::flatten_to(T* out_arr) { - FLS_ASSERT_NOT_NULL_POINTER(out_arr) - FLS_ASSERT_NOT_NULL_POINTER(buf_arr[0].data()) - FLS_ASSERT_NOT_NULL_POINTER(buf_arr[1].data()) - - auto* val_arr = reinterpret_cast(buf_arr[0].mutable_data()); - auto* pos_arr = reinterpret_cast(buf_arr[1].mutable_data()); - - for (n_t i {0}; i < vec_n_tup(); ++i) { - out_arr[i] = val_arr[pos_arr[i]]; - } - - FLS_PLOG_KEY_VALUE("val_arr", ToStr::to_str(val_arr)); - FLS_PLOG_KEY_VALUE("pos_arr", ToStr::to_str(pos_arr)); -} - -Vec::~Vec() = default; - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -// clang-format off -template void Vec::Init(const uint8_t *p,EngineState& stt); -template void Vec::Init(const uint16_t *p,EngineState& stt); -template void Vec::Init(const uint32_t *p,EngineState& stt); -template void Vec::Init(const uint64_t *p,EngineState& stt); -template void Vec::Init(const int8_t *p,EngineState& stt); -template void Vec::Init(const int16_t *p,EngineState& stt); -template void Vec::Init(const int32_t *p,EngineState& stt); -template void Vec::Init(const int64_t *p,EngineState& stt); -template void Vec::Init(const float *p,EngineState& stt); -template void Vec::Init(const double *p,EngineState& stt); -template void Vec::Init(const bool *p,EngineState& stt); -template void Vec::Init(const str_pt *p,EngineState& stt); -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -// clang-format off -template void Vec::flatten_to(dbl_pt *out_p); -template void Vec::flatten_to(uint64_t *out_p); -template void Vec::flatten_to(uint32_t *out_p); -template void Vec::flatten_to(uint16_t *out_p); -template void Vec::flatten_to(uint8_t *out_p); - -} // namespace fastlanes diff --git a/src/cor/mtd.cpp b/src/cor/mtd.cpp deleted file mode 100644 index e205f799..00000000 --- a/src/cor/mtd.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "fls/cor/mtd.hpp" -#include "fls/cfg/cfg.hpp" - -namespace fastlanes { -template -Mtd::Mtd(n_t cap) - : cap {cap} - , size {0} - , compressed_sz {0} - , min_max {} - , histogram {} { -} - -template -n_t Mtd::Count() const { - return size.Count(); -} - -template -double Mtd::Ratio() const { - return size / compressed_sz; -} - -template -n_t Mtd::UnusedSize() const { - return size - compressed_sz; -} - -template -T Mtd::Min() const { - return min_max.min; -} - -template -T Mtd::Max() const { - return min_max.max; -} - -template -void Mtd::Reset() { - min_max.Reset(); - histogram.Reset(); - compressed_sz.reset(); -} - -template -n_t Mtd::VecCount() const { - return Count() / fastlanes::CFG::VEC_TUP_C; -} - -template -double Mtd::Occupancy() const { - return compressed_sz / cap; -} - -template -n_t Mtd::Cardinality() const { - FLS_ASSERT(histogram.val_vec.size() != 0, "Hist.size is zero. Call cal before use!", " "); - - return histogram.val_vec.size(); -} - -template -n_t Mtd::Size() const { - return size.Val(); -} -template -void Mtd::Reset(n_t c) { - min_max.Reset(); - histogram.Reset(); - compressed_sz.reset(); - size = byte_c {c * sizeof(T)}; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(Mtd) -// CHECK -// { -// template class Mtd; -// template class Mtd; -// template class Mtd; -// } -} // namespace fastlanes diff --git a/src/cor/prm/CMakeLists.txt b/src/cor/prm/CMakeLists.txt index d5ff4f03..17761234 100644 --- a/src/cor/prm/CMakeLists.txt +++ b/src/cor/prm/CMakeLists.txt @@ -3,8 +3,7 @@ add_subdirectory(fsst12) add_library(fls_all_prms OBJECT - prm.cpp - type.cpp) + prm.cpp) set(FASTLANES_OBJECT_FILES ${FASTLANES_OBJECT_FILES} $ diff --git a/src/cor/prm/alp/CMakeLists.txt b/src/cor/prm/alp/CMakeLists.txt deleted file mode 100644 index 85deac7e..00000000 --- a/src/cor/prm/alp/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -add_library(fls_alp_prm - OBJECT - alp_decompress.cpp - alp_compress.cpp) - -if (ENABLE_IWYU) - set_target_properties(fls_alp_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_alp_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -target_link_libraries(fls_alp_prm - PUBLIC - FastLanes::headers -) diff --git a/src/cor/prm/alp/alp_compress.cpp b/src/cor/prm/alp/alp_compress.cpp deleted file mode 100644 index ca67d822..00000000 --- a/src/cor/prm/alp/alp_compress.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "alp.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/engine.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/alp_prm.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/primitive/ffor/ffor.hpp" - -namespace fastlanes { -static void alp_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - - // auto* dbl_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - auto& alp_stt = *stt.eng_state.analyze_state_up->alp_dbl_state_up; - - // auto* exc_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 0].mutable_data()); - // auto* pos_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 1].mutable_data()); - auto* exc_c_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 2].mutable_data()); - auto* base_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 3].mutable_data()); - auto* ffor_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 5].mutable_data()); - - uint8_t bw = 0; - int64_t tmp_dig_arr[vec_sz()]; // TODO BUFFER_POOl - // alp::encoder::encode(dbl_arr, exc_arr, pos_arr, exc_c_arr, tmp_dig_arr, alp_stt); TODO - alp::encoder::analyze_ffor(tmp_dig_arr, bw, base_arr); - generated::ffor::fallback::scalar::ffor(tmp_dig_arr, ffor_arr, bw, base_arr); - - auto exc_c = exc_c_arr[0]; - alp_mtd_t alp_mtd {bw, alp_stt.exp, alp_stt.fac}; - - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAdvance(exc_c * sizeof(uint16_t)); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAdvance(exc_c * sizeof(uint16_t)); - des_vec.buf_arr[stt.cur_des_buff + 2].UnsafeAdvance(sizeof(uint16_t)); - des_vec.buf_arr[stt.cur_des_buff + 3].UnsafeAdvance(sizeof(int64_t)); - des_vec.buf_arr[stt.cur_des_buff + 4].UnsafeAppend(&alp_mtd, sizeof(alp_mtd_t)); - des_vec.buf_arr[stt.cur_des_buff + 5].UnsafeAdvance(ffor_prm::vec_sz(bw)); - - stt.cur_des_buff += 6; -} - -template -cmpr_fun_t alp_prm::ResolveCompressFunc() { - if constexpr (std::is_same_v) { - return alp_compress; - } else { - FLS_ABORT("ONLY WORKS FOR DOUBLE") - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, alp_prm::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/alp/alp_decompress.cpp b/src/cor/prm/alp/alp_decompress.cpp deleted file mode 100644 index 58a1621c..00000000 --- a/src/cor/prm/alp/alp_decompress.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "alp.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/alp_prm.hpp" -#include "fls/primitive/unffor/unffor.hpp" - -namespace fastlanes { -static void alp_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 5]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 4]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 3]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 2]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 1]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 0]) - - auto* ffor_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* alp_mtd_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* base_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 2]); - // auto* exc_c_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 3]); - // auto* pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 4]); - // auto* exc_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 5]); - - auto* dec_dbl_arr = reinterpret_cast(des.arr[stt.cur_des_arr]); - - int64_t unffor_arr[vec_sz()]; // TODO BUFFER_POOl - auto alp_mtd = alp_mtd_arr[0]; - - generated::unffor::fallback::scalar::unffor(ffor_arr, unffor_arr, alp_mtd.bw, base_arr); - alp::decoder::decode(unffor_arr, alp_mtd.fac, alp_mtd.exp, dec_dbl_arr); - // alp::decoder::patch_exceptions(dec_dbl_arr, exc_arr, pos_arr, exc_c_arr); TODO - - stt.cur_src_arr -= 6; -} - -template -de_cmpr_fun_t alp_prm::ResolveDecompressFunc() { - if constexpr (std::is_same_v) { - return alp_decompress; - } else { - FLS_ABORT("ONLY WORKS FOR DOUBLE") - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, alp_prm::ResolveDecompressFunc) -} // namespace fastlanes diff --git a/src/cor/prm/b_ffor/CMakeLists.txt b/src/cor/prm/b_ffor/CMakeLists.txt deleted file mode 100644 index 7c91d916..00000000 --- a/src/cor/prm/b_ffor/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -add_library(fls_b_ffor_prm - OBJECT - b_ffor_decompress.cpp - b_ffor_compress.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_b_ffor_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_b_ffor_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -target_link_libraries(fls_b_ffor_prm - PUBLIC - FastLanes::headers -) diff --git a/src/cor/prm/b_ffor/b_ffor_compress.cpp b/src/cor/prm/b_ffor/b_ffor_compress.cpp deleted file mode 100644 index 52d12ed3..00000000 --- a/src/cor/prm/b_ffor/b_ffor_compress.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/b_ffor_prm.hpp" -#include "fls/ffor.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void ffor_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { -} - -template -cmpr_fun_t b_ffor_prm::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, b_ffor_prm::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/b_ffor/b_ffor_decompress.cpp b/src/cor/prm/b_ffor/b_ffor_decompress.cpp deleted file mode 100644 index e60b2315..00000000 --- a/src/cor/prm/b_ffor/b_ffor_decompress.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/b_ffor_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/unffor/unffor.hpp" - -namespace fastlanes { -template -static void b_ffor_decompress(PageParam src, VecParam des, DecompressState& stt) { -} - -template -de_cmpr_fun_t b_ffor_prm::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc) -// FLS_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc, parquet::FixedLenByteArray) -// FLS_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc, parquet::Int96) -// FLS_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc, parquet::ByteArray) -} // namespace fastlanes diff --git a/src/cor/prm/bitpack/CMakeLists.txt b/src/cor/prm/bitpack/CMakeLists.txt deleted file mode 100644 index 87f9358c..00000000 --- a/src/cor/prm/bitpack/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -add_library(fls_bitpack - OBJECT - bitpack_decompress.cpp - bitpack_compress.cpp) - -target_compile_definitions(fls_bitpack PRIVATE IS_SCALAR) -target_compile_options(fls_bitpack PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_bitpack PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_bitpack PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -target_link_libraries(fls_bitpack - PUBLIC - FastLanes::headers -) diff --git a/src/cor/prm/bitpack/bitpack_compress.cpp b/src/cor/prm/bitpack/bitpack_compress.cpp deleted file mode 100644 index 19cd74a0..00000000 --- a/src/cor/prm/bitpack/bitpack_compress.cpp +++ /dev/null @@ -1,22254 +0,0 @@ -#include "fls/cor/prm/bitpack.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t -#include // for __restrict -namespace fastlanes { namespace bitpack { -void pack_1bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 1U; - src = *(in + 128 * 2 + i); - tmp |= src << 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 3U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 5U; - src = *(in + 128 * 6 + i); - tmp |= src << 6U; - src = *(in + 128 * 7 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 2U; - src = *(in + 128 * 2 + i); - tmp |= src << 4U; - src = *(in + 128 * 3 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - src = *(in + 128 * 5 + i); - tmp |= src << 2U; - src = *(in + 128 * 6 + i); - tmp |= src << 4U; - src = *(in + 128 * 7 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_3bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 3U; - src = *(in + 128 * 2 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src >> 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 1U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src >> 1U; - src = *(in + 128 * 6 + i); - tmp |= src << 2U; - src = *(in + 128 * 7 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_4bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src; - src = *(in + 128 * 3 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - src = *(in + 128 * 5 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src; - src = *(in + 128 * 7 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_5bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src >> 3U; - src = *(in + 128 * 2 + i); - tmp |= src << 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 3 + i); - tmp = src >> 1U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src >> 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 1U; - src = *(in + 128 * 6 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src >> 2U; - src = *(in + 128 * 7 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_6bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src >> 2U; - src = *(in + 128 * 2 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src >> 4U; - src = *(in + 128 * 3 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - src = *(in + 128 * 5 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src >> 2U; - src = *(in + 128 * 6 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src >> 4U; - src = *(in + 128 * 7 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_7bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src >> 1U; - src = *(in + 128 * 2 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src >> 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 3 + i); - tmp = src >> 3U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src >> 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src >> 5U; - src = *(in + 128 * 6 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src >> 6U; - src = *(in + 128 * 7 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_8bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 7 + i); - tmp = src; - *(out + i) = tmp; - out -= 896; - } -} -void pack_1bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 1U; - src = *(in + 64 * 2 + i); - tmp |= src << 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 3U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 5U; - src = *(in + 64 * 6 + i); - tmp |= src << 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 7U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 9U; - src = *(in + 64 * 10 + i); - tmp |= src << 10U; - src = *(in + 64 * 11 + i); - tmp |= src << 11U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - src = *(in + 64 * 13 + i); - tmp |= src << 13U; - src = *(in + 64 * 14 + i); - tmp |= src << 14U; - src = *(in + 64 * 15 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 2U; - src = *(in + 64 * 2 + i); - tmp |= src << 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 6U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 10U; - src = *(in + 64 * 6 + i); - tmp |= src << 12U; - src = *(in + 64 * 7 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 2U; - src = *(in + 64 * 10 + i); - tmp |= src << 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 6U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 10U; - src = *(in + 64 * 14 + i); - tmp |= src << 12U; - src = *(in + 64 * 15 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out -= 64; - } -} -void pack_3bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 3U; - src = *(in + 64 * 2 + i); - tmp |= src << 6U; - src = *(in + 64 * 3 + i); - tmp |= src << 9U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - src = *(in + 64 * 5 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 1U; - src = *(in + 64 * 6 + i); - tmp |= src << 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 5U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 11U; - src = *(in + 64 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 1U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 7U; - src = *(in + 64 * 14 + i); - tmp |= src << 10U; - src = *(in + 64 * 15 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_4bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 4U; - src = *(in + 64 * 2 + i); - tmp |= src << 8U; - src = *(in + 64 * 3 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - src = *(in + 64 * 5 + i); - tmp |= src << 4U; - src = *(in + 64 * 6 + i); - tmp |= src << 8U; - src = *(in + 64 * 7 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 4U; - src = *(in + 64 * 10 + i); - tmp |= src << 8U; - src = *(in + 64 * 11 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - src = *(in + 64 * 13 + i); - tmp |= src << 4U; - src = *(in + 64 * 14 + i); - tmp |= src << 8U; - src = *(in + 64 * 15 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out -= 192; - } -} -void pack_5bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 5U; - src = *(in + 64 * 2 + i); - tmp |= src << 10U; - src = *(in + 64 * 3 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 1U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 9U; - src = *(in + 64 * 6 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 3U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 3U; - src = *(in + 64 * 10 + i); - tmp |= src << 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 7U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 1U; - src = *(in + 64 * 14 + i); - tmp |= src << 6U; - src = *(in + 64 * 15 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_6bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 6U; - src = *(in + 64 * 2 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 2U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 2U; - src = *(in + 64 * 6 + i); - tmp |= src << 4U; - src = *(in + 64 * 7 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 6U; - src = *(in + 64 * 10 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 2U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 2U; - src = *(in + 64 * 14 + i); - tmp |= src << 4U; - src = *(in + 64 * 15 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out -= 320; - } -} -void pack_7bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 7U; - src = *(in + 64 * 2 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 5U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 3U; - src = *(in + 64 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 1U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 1U; - src = *(in + 64 * 10 + i); - tmp |= src << 6U; - src = *(in + 64 * 11 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 3U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 5U; - src = *(in + 64 * 14 + i); - tmp |= src << 2U; - src = *(in + 64 * 15 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_8bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src; - src = *(in + 64 * 3 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - src = *(in + 64 * 5 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src; - src = *(in + 64 * 7 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src; - src = *(in + 64 * 11 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - src = *(in + 64 * 13 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src; - src = *(in + 64 * 15 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out -= 448; - } -} -void pack_9bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 7U; - src = *(in + 64 * 2 + i); - tmp |= src << 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 5U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 3U; - src = *(in + 64 * 6 + i); - tmp |= src << 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 1U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 1U; - src = *(in + 64 * 10 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 6U; - src = *(in + 64 * 11 + i); - tmp |= src << 3U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 5U; - src = *(in + 64 * 14 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 2U; - src = *(in + 64 * 15 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_10bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 6U; - src = *(in + 64 * 2 + i); - tmp |= src << 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 2U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 2U; - src = *(in + 64 * 6 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 4U; - src = *(in + 64 * 7 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 6U; - src = *(in + 64 * 10 + i); - tmp |= src << 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 2U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 2U; - src = *(in + 64 * 14 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 4U; - src = *(in + 64 * 15 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 576; - } -} -void pack_11bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 5U; - src = *(in + 64 * 2 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 10U; - src = *(in + 64 * 3 + i); - tmp |= src << 1U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 9U; - src = *(in + 64 * 6 + i); - tmp |= src << 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 3U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 3U; - src = *(in + 64 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 7U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 1U; - src = *(in + 64 * 14 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 6U; - src = *(in + 64 * 15 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_12bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 4U; - src = *(in + 64 * 2 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 8U; - src = *(in + 64 * 3 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - src = *(in + 64 * 5 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 4U; - src = *(in + 64 * 6 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 8U; - src = *(in + 64 * 7 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 4U; - src = *(in + 64 * 10 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 8U; - src = *(in + 64 * 11 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - src = *(in + 64 * 13 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 4U; - src = *(in + 64 * 14 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 8U; - src = *(in + 64 * 15 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 704; - } -} -void pack_13bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 3U; - src = *(in + 64 * 2 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 6U; - src = *(in + 64 * 3 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 9U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 12U; - src = *(in + 64 * 5 + i); - tmp |= src << 1U; - src = *(in + 64 * 6 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 5U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 11U; - src = *(in + 64 * 10 + i); - tmp |= src << 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 1U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 7U; - src = *(in + 64 * 14 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 10U; - src = *(in + 64 * 15 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_14bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 2U; - src = *(in + 64 * 2 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 6U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 10U; - src = *(in + 64 * 6 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 12U; - src = *(in + 64 * 7 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 2U; - src = *(in + 64 * 10 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 6U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 10U; - src = *(in + 64 * 14 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 12U; - src = *(in + 64 * 15 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 832; - } -} -void pack_15bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 1U; - src = *(in + 64 * 2 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 3U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 5U; - src = *(in + 64 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 7U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 9U; - src = *(in + 64 * 10 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 10U; - src = *(in + 64 * 11 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 11U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 12U; - src = *(in + 64 * 13 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 13U; - src = *(in + 64 * 14 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 14U; - src = *(in + 64 * 15 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 896; - } -} -void pack_16bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 15 + i); - tmp = src; - *(out + i) = tmp; - out -= 960; - } -} -void pack_1bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 1U; - src = *(in + 32 * 2 + i); - tmp |= src << 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 3U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 5U; - src = *(in + 32 * 6 + i); - tmp |= src << 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 7U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 9U; - src = *(in + 32 * 10 + i); - tmp |= src << 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 11U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 13U; - src = *(in + 32 * 14 + i); - tmp |= src << 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 15U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 17U; - src = *(in + 32 * 18 + i); - tmp |= src << 18U; - src = *(in + 32 * 19 + i); - tmp |= src << 19U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 21U; - src = *(in + 32 * 22 + i); - tmp |= src << 22U; - src = *(in + 32 * 23 + i); - tmp |= src << 23U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 25U; - src = *(in + 32 * 26 + i); - tmp |= src << 26U; - src = *(in + 32 * 27 + i); - tmp |= src << 27U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - src = *(in + 32 * 29 + i); - tmp |= src << 29U; - src = *(in + 32 * 30 + i); - tmp |= src << 30U; - src = *(in + 32 * 31 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 2U; - src = *(in + 32 * 2 + i); - tmp |= src << 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 6U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 10U; - src = *(in + 32 * 6 + i); - tmp |= src << 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 14U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 18U; - src = *(in + 32 * 10 + i); - tmp |= src << 20U; - src = *(in + 32 * 11 + i); - tmp |= src << 22U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - src = *(in + 32 * 13 + i); - tmp |= src << 26U; - src = *(in + 32 * 14 + i); - tmp |= src << 28U; - src = *(in + 32 * 15 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 2U; - src = *(in + 32 * 18 + i); - tmp |= src << 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 6U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 10U; - src = *(in + 32 * 22 + i); - tmp |= src << 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 14U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 18U; - src = *(in + 32 * 26 + i); - tmp |= src << 20U; - src = *(in + 32 * 27 + i); - tmp |= src << 22U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - src = *(in + 32 * 29 + i); - tmp |= src << 26U; - src = *(in + 32 * 30 + i); - tmp |= src << 28U; - src = *(in + 32 * 31 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out -= 32; - } -} -void pack_3bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 3U; - src = *(in + 32 * 2 + i); - tmp |= src << 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 9U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 15U; - src = *(in + 32 * 6 + i); - tmp |= src << 18U; - src = *(in + 32 * 7 + i); - tmp |= src << 21U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 27U; - src = *(in + 32 * 10 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 1U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 7U; - src = *(in + 32 * 14 + i); - tmp |= src << 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 13U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 19U; - src = *(in + 32 * 18 + i); - tmp |= src << 22U; - src = *(in + 32 * 19 + i); - tmp |= src << 25U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - src = *(in + 32 * 21 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 1U; - src = *(in + 32 * 22 + i); - tmp |= src << 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 5U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 11U; - src = *(in + 32 * 26 + i); - tmp |= src << 14U; - src = *(in + 32 * 27 + i); - tmp |= src << 17U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 23U; - src = *(in + 32 * 30 + i); - tmp |= src << 26U; - src = *(in + 32 * 31 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out -= 64; - } -} -void pack_4bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 4U; - src = *(in + 32 * 2 + i); - tmp |= src << 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 12U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 20U; - src = *(in + 32 * 6 + i); - tmp |= src << 24U; - src = *(in + 32 * 7 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 4U; - src = *(in + 32 * 10 + i); - tmp |= src << 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 12U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 20U; - src = *(in + 32 * 14 + i); - tmp |= src << 24U; - src = *(in + 32 * 15 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 4U; - src = *(in + 32 * 18 + i); - tmp |= src << 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 12U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 20U; - src = *(in + 32 * 22 + i); - tmp |= src << 24U; - src = *(in + 32 * 23 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 4U; - src = *(in + 32 * 26 + i); - tmp |= src << 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 12U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 20U; - src = *(in + 32 * 30 + i); - tmp |= src << 24U; - src = *(in + 32 * 31 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out -= 96; - } -} -void pack_5bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 5U; - src = *(in + 32 * 2 + i); - tmp |= src << 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 15U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - src = *(in + 32 * 5 + i); - tmp |= src << 25U; - src = *(in + 32 * 6 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 3U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 13U; - src = *(in + 32 * 10 + i); - tmp |= src << 18U; - src = *(in + 32 * 11 + i); - tmp |= src << 23U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 1U; - src = *(in + 32 * 14 + i); - tmp |= src << 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 11U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 21U; - src = *(in + 32 * 18 + i); - tmp |= src << 26U; - src = *(in + 32 * 19 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 1U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 9U; - src = *(in + 32 * 22 + i); - tmp |= src << 14U; - src = *(in + 32 * 23 + i); - tmp |= src << 19U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 3U; - src = *(in + 32 * 26 + i); - tmp |= src << 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 7U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 17U; - src = *(in + 32 * 30 + i); - tmp |= src << 22U; - src = *(in + 32 * 31 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_6bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 6U; - src = *(in + 32 * 2 + i); - tmp |= src << 12U; - src = *(in + 32 * 3 + i); - tmp |= src << 18U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - src = *(in + 32 * 5 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 2U; - src = *(in + 32 * 6 + i); - tmp |= src << 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 10U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 22U; - src = *(in + 32 * 10 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 2U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 14U; - src = *(in + 32 * 14 + i); - tmp |= src << 20U; - src = *(in + 32 * 15 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 6U; - src = *(in + 32 * 18 + i); - tmp |= src << 12U; - src = *(in + 32 * 19 + i); - tmp |= src << 18U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - src = *(in + 32 * 21 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 2U; - src = *(in + 32 * 22 + i); - tmp |= src << 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 10U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 22U; - src = *(in + 32 * 26 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 2U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 14U; - src = *(in + 32 * 30 + i); - tmp |= src << 20U; - src = *(in + 32 * 31 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out -= 160; - } -} -void pack_7bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 7U; - src = *(in + 32 * 2 + i); - tmp |= src << 14U; - src = *(in + 32 * 3 + i); - tmp |= src << 21U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 3U; - src = *(in + 32 * 6 + i); - tmp |= src << 10U; - src = *(in + 32 * 7 + i); - tmp |= src << 17U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 1U; - src = *(in + 32 * 10 + i); - tmp |= src << 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 13U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - src = *(in + 32 * 13 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 5U; - src = *(in + 32 * 14 + i); - tmp |= src << 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 9U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 23U; - src = *(in + 32 * 18 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 5U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 19U; - src = *(in + 32 * 22 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 1U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 15U; - src = *(in + 32 * 26 + i); - tmp |= src << 22U; - src = *(in + 32 * 27 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 3U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 11U; - src = *(in + 32 * 30 + i); - tmp |= src << 18U; - src = *(in + 32 * 31 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out -= 192; - } -} -void pack_8bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 8U; - src = *(in + 32 * 2 + i); - tmp |= src << 16U; - src = *(in + 32 * 3 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - src = *(in + 32 * 5 + i); - tmp |= src << 8U; - src = *(in + 32 * 6 + i); - tmp |= src << 16U; - src = *(in + 32 * 7 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 8U; - src = *(in + 32 * 10 + i); - tmp |= src << 16U; - src = *(in + 32 * 11 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - src = *(in + 32 * 13 + i); - tmp |= src << 8U; - src = *(in + 32 * 14 + i); - tmp |= src << 16U; - src = *(in + 32 * 15 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 8U; - src = *(in + 32 * 18 + i); - tmp |= src << 16U; - src = *(in + 32 * 19 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - src = *(in + 32 * 21 + i); - tmp |= src << 8U; - src = *(in + 32 * 22 + i); - tmp |= src << 16U; - src = *(in + 32 * 23 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 8U; - src = *(in + 32 * 26 + i); - tmp |= src << 16U; - src = *(in + 32 * 27 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - src = *(in + 32 * 29 + i); - tmp |= src << 8U; - src = *(in + 32 * 30 + i); - tmp |= src << 16U; - src = *(in + 32 * 31 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out -= 224; - } -} -void pack_9bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 9U; - src = *(in + 32 * 2 + i); - tmp |= src << 18U; - src = *(in + 32 * 3 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 5U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 13U; - src = *(in + 32 * 6 + i); - tmp |= src << 22U; - src = *(in + 32 * 7 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 1U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 17U; - src = *(in + 32 * 10 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 3U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 21U; - src = *(in + 32 * 14 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 7U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 7U; - src = *(in + 32 * 18 + i); - tmp |= src << 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 11U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 3U; - src = *(in + 32 * 22 + i); - tmp |= src << 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 15U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 1U; - src = *(in + 32 * 26 + i); - tmp |= src << 10U; - src = *(in + 32 * 27 + i); - tmp |= src << 19U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 5U; - src = *(in + 32 * 30 + i); - tmp |= src << 14U; - src = *(in + 32 * 31 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_10bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 10U; - src = *(in + 32 * 2 + i); - tmp |= src << 20U; - src = *(in + 32 * 3 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 2U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 18U; - src = *(in + 32 * 6 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 6U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 6U; - src = *(in + 32 * 10 + i); - tmp |= src << 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 14U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 2U; - src = *(in + 32 * 14 + i); - tmp |= src << 12U; - src = *(in + 32 * 15 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 10U; - src = *(in + 32 * 18 + i); - tmp |= src << 20U; - src = *(in + 32 * 19 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 2U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 18U; - src = *(in + 32 * 22 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 6U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 6U; - src = *(in + 32 * 26 + i); - tmp |= src << 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 14U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 2U; - src = *(in + 32 * 30 + i); - tmp |= src << 12U; - src = *(in + 32 * 31 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out -= 288; - } -} -void pack_11bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 11U; - src = *(in + 32 * 2 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 1U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 9U; - src = *(in + 32 * 6 + i); - tmp |= src << 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 13U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 3U; - src = *(in + 32 * 10 + i); - tmp |= src << 14U; - src = *(in + 32 * 11 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 7U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 15U; - src = *(in + 32 * 14 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 5U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 5U; - src = *(in + 32 * 18 + i); - tmp |= src << 6U; - src = *(in + 32 * 19 + i); - tmp |= src << 17U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 7U; - src = *(in + 32 * 22 + i); - tmp |= src << 18U; - src = *(in + 32 * 23 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 3U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 19U; - src = *(in + 32 * 26 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 9U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 1U; - src = *(in + 32 * 30 + i); - tmp |= src << 10U; - src = *(in + 32 * 31 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out -= 320; - } -} -void pack_12bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 12U; - src = *(in + 32 * 2 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 4U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 4U; - src = *(in + 32 * 6 + i); - tmp |= src << 8U; - src = *(in + 32 * 7 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 12U; - src = *(in + 32 * 10 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 4U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 4U; - src = *(in + 32 * 14 + i); - tmp |= src << 8U; - src = *(in + 32 * 15 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 12U; - src = *(in + 32 * 18 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 4U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 4U; - src = *(in + 32 * 22 + i); - tmp |= src << 8U; - src = *(in + 32 * 23 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 12U; - src = *(in + 32 * 26 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 4U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 4U; - src = *(in + 32 * 30 + i); - tmp |= src << 8U; - src = *(in + 32 * 31 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out -= 352; - } -} -void pack_13bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 13U; - src = *(in + 32 * 2 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 7U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 1U; - src = *(in + 32 * 6 + i); - tmp |= src << 14U; - src = *(in + 32 * 7 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 5U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 11U; - src = *(in + 32 * 10 + i); - tmp |= src << 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 15U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 9U; - src = *(in + 32 * 14 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 3U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 3U; - src = *(in + 32 * 18 + i); - tmp |= src << 10U; - src = *(in + 32 * 19 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 9U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 17U; - src = *(in + 32 * 22 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 11U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 5U; - src = *(in + 32 * 26 + i); - tmp |= src << 18U; - src = *(in + 32 * 27 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 1U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 7U; - src = *(in + 32 * 30 + i); - tmp |= src << 6U; - src = *(in + 32 * 31 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_14bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 14U; - src = *(in + 32 * 2 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 10U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 6U; - src = *(in + 32 * 6 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 2U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 2U; - src = *(in + 32 * 10 + i); - tmp |= src << 12U; - src = *(in + 32 * 11 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 6U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 10U; - src = *(in + 32 * 14 + i); - tmp |= src << 4U; - src = *(in + 32 * 15 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 14U; - src = *(in + 32 * 18 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 10U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 6U; - src = *(in + 32 * 22 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 2U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 2U; - src = *(in + 32 * 26 + i); - tmp |= src << 12U; - src = *(in + 32 * 27 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 6U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 10U; - src = *(in + 32 * 30 + i); - tmp |= src << 4U; - src = *(in + 32 * 31 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out -= 416; - } -} -void pack_15bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 15U; - src = *(in + 32 * 2 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 13U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 11U; - src = *(in + 32 * 6 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 9U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 7U; - src = *(in + 32 * 10 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 5U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 3U; - src = *(in + 32 * 14 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 1U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 1U; - src = *(in + 32 * 18 + i); - tmp |= src << 14U; - src = *(in + 32 * 19 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 3U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 5U; - src = *(in + 32 * 22 + i); - tmp |= src << 10U; - src = *(in + 32 * 23 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 7U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 9U; - src = *(in + 32 * 26 + i); - tmp |= src << 6U; - src = *(in + 32 * 27 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 11U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 13U; - src = *(in + 32 * 30 + i); - tmp |= src << 2U; - src = *(in + 32 * 31 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out -= 448; - } -} -void pack_16bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src; - src = *(in + 32 * 3 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - src = *(in + 32 * 5 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src; - src = *(in + 32 * 7 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src; - src = *(in + 32 * 11 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - src = *(in + 32 * 13 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src; - src = *(in + 32 * 15 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src; - src = *(in + 32 * 19 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - src = *(in + 32 * 21 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src; - src = *(in + 32 * 23 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src; - src = *(in + 32 * 27 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - src = *(in + 32 * 29 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src; - src = *(in + 32 * 31 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out -= 480; - } -} -void pack_17bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 15U; - src = *(in + 32 * 2 + i); - tmp |= src << 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 13U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 11U; - src = *(in + 32 * 6 + i); - tmp |= src << 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 9U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 7U; - src = *(in + 32 * 10 + i); - tmp |= src << 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 5U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 3U; - src = *(in + 32 * 14 + i); - tmp |= src << 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 1U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 1U; - src = *(in + 32 * 18 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 14U; - src = *(in + 32 * 19 + i); - tmp |= src << 3U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 5U; - src = *(in + 32 * 22 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 10U; - src = *(in + 32 * 23 + i); - tmp |= src << 7U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 9U; - src = *(in + 32 * 26 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 6U; - src = *(in + 32 * 27 + i); - tmp |= src << 11U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 13U; - src = *(in + 32 * 30 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 2U; - src = *(in + 32 * 31 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_18bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 14U; - src = *(in + 32 * 2 + i); - tmp |= src << 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 10U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 6U; - src = *(in + 32 * 6 + i); - tmp |= src << 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 2U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 2U; - src = *(in + 32 * 10 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 12U; - src = *(in + 32 * 11 + i); - tmp |= src << 6U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 10U; - src = *(in + 32 * 14 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 4U; - src = *(in + 32 * 15 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 14U; - src = *(in + 32 * 18 + i); - tmp |= src << 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 10U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 6U; - src = *(in + 32 * 22 + i); - tmp |= src << 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 2U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 2U; - src = *(in + 32 * 26 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 12U; - src = *(in + 32 * 27 + i); - tmp |= src << 6U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 10U; - src = *(in + 32 * 30 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 4U; - src = *(in + 32 * 31 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out -= 544; - } -} -void pack_19bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 13U; - src = *(in + 32 * 2 + i); - tmp |= src << 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 7U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 1U; - src = *(in + 32 * 6 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 14U; - src = *(in + 32 * 7 + i); - tmp |= src << 5U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 11U; - src = *(in + 32 * 10 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 15U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 9U; - src = *(in + 32 * 14 + i); - tmp |= src << 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 3U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 3U; - src = *(in + 32 * 18 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 10U; - src = *(in + 32 * 19 + i); - tmp |= src << 9U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 17U; - src = *(in + 32 * 22 + i); - tmp |= src << 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 11U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 5U; - src = *(in + 32 * 26 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 18U; - src = *(in + 32 * 27 + i); - tmp |= src << 1U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 7U; - src = *(in + 32 * 30 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 6U; - src = *(in + 32 * 31 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out -= 576; - } -} -void pack_20bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 12U; - src = *(in + 32 * 2 + i); - tmp |= src << 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 4U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 4U; - src = *(in + 32 * 6 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 8U; - src = *(in + 32 * 7 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 12U; - src = *(in + 32 * 10 + i); - tmp |= src << 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 4U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 4U; - src = *(in + 32 * 14 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 8U; - src = *(in + 32 * 15 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 12U; - src = *(in + 32 * 18 + i); - tmp |= src << 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 4U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 4U; - src = *(in + 32 * 22 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 8U; - src = *(in + 32 * 23 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 12U; - src = *(in + 32 * 26 + i); - tmp |= src << 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 4U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 4U; - src = *(in + 32 * 30 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 8U; - src = *(in + 32 * 31 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out -= 608; - } -} -void pack_21bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 11U; - src = *(in + 32 * 2 + i); - tmp |= src << 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 1U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 9U; - src = *(in + 32 * 6 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 13U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 3U; - src = *(in + 32 * 10 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 14U; - src = *(in + 32 * 11 + i); - tmp |= src << 7U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 15U; - src = *(in + 32 * 14 + i); - tmp |= src << 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 5U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 5U; - src = *(in + 32 * 18 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 6U; - src = *(in + 32 * 19 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 17U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 7U; - src = *(in + 32 * 22 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 18U; - src = *(in + 32 * 23 + i); - tmp |= src << 3U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 19U; - src = *(in + 32 * 26 + i); - tmp |= src << 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 9U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 1U; - src = *(in + 32 * 30 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 10U; - src = *(in + 32 * 31 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_22bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 10U; - src = *(in + 32 * 2 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 20U; - src = *(in + 32 * 3 + i); - tmp |= src << 2U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 18U; - src = *(in + 32 * 6 + i); - tmp |= src << 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 6U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 6U; - src = *(in + 32 * 10 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 14U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 2U; - src = *(in + 32 * 14 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 12U; - src = *(in + 32 * 15 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 10U; - src = *(in + 32 * 18 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 20U; - src = *(in + 32 * 19 + i); - tmp |= src << 2U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 18U; - src = *(in + 32 * 22 + i); - tmp |= src << 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 6U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 6U; - src = *(in + 32 * 26 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 14U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 2U; - src = *(in + 32 * 30 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 12U; - src = *(in + 32 * 31 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out -= 672; - } -} -void pack_23bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 9U; - src = *(in + 32 * 2 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 18U; - src = *(in + 32 * 3 + i); - tmp |= src << 5U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 13U; - src = *(in + 32 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 22U; - src = *(in + 32 * 7 + i); - tmp |= src << 1U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 17U; - src = *(in + 32 * 10 + i); - tmp |= src << 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 3U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 21U; - src = *(in + 32 * 14 + i); - tmp |= src << 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 7U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 7U; - src = *(in + 32 * 18 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 11U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 3U; - src = *(in + 32 * 22 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 15U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 1U; - src = *(in + 32 * 26 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 10U; - src = *(in + 32 * 27 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 19U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 5U; - src = *(in + 32 * 30 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 14U; - src = *(in + 32 * 31 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out -= 704; - } -} -void pack_24bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 8U; - src = *(in + 32 * 2 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 16U; - src = *(in + 32 * 3 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - src = *(in + 32 * 5 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 8U; - src = *(in + 32 * 6 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 16U; - src = *(in + 32 * 7 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 8U; - src = *(in + 32 * 10 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 16U; - src = *(in + 32 * 11 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - src = *(in + 32 * 13 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 8U; - src = *(in + 32 * 14 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 16U; - src = *(in + 32 * 15 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 8U; - src = *(in + 32 * 18 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 16U; - src = *(in + 32 * 19 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - src = *(in + 32 * 21 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 8U; - src = *(in + 32 * 22 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 16U; - src = *(in + 32 * 23 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 8U; - src = *(in + 32 * 26 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 16U; - src = *(in + 32 * 27 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - src = *(in + 32 * 29 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 8U; - src = *(in + 32 * 30 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 16U; - src = *(in + 32 * 31 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out -= 736; - } -} -void pack_25bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 7U; - src = *(in + 32 * 2 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 14U; - src = *(in + 32 * 3 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 21U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 3U; - src = *(in + 32 * 6 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 10U; - src = *(in + 32 * 7 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 17U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 1U; - src = *(in + 32 * 10 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 13U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 20U; - src = *(in + 32 * 13 + i); - tmp |= src << 5U; - src = *(in + 32 * 14 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 9U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 23U; - src = *(in + 32 * 18 + i); - tmp |= src << 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 5U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 19U; - src = *(in + 32 * 22 + i); - tmp |= src << 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 1U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 15U; - src = *(in + 32 * 26 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 22U; - src = *(in + 32 * 27 + i); - tmp |= src << 3U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 11U; - src = *(in + 32 * 30 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 18U; - src = *(in + 32 * 31 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_26bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 6U; - src = *(in + 32 * 2 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 12U; - src = *(in + 32 * 3 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 18U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 24U; - src = *(in + 32 * 5 + i); - tmp |= src << 2U; - src = *(in + 32 * 6 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 10U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 22U; - src = *(in + 32 * 10 + i); - tmp |= src << 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 2U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 14U; - src = *(in + 32 * 14 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 20U; - src = *(in + 32 * 15 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 6U; - src = *(in + 32 * 18 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 12U; - src = *(in + 32 * 19 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 18U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 24U; - src = *(in + 32 * 21 + i); - tmp |= src << 2U; - src = *(in + 32 * 22 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 10U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 22U; - src = *(in + 32 * 26 + i); - tmp |= src << 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 2U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 14U; - src = *(in + 32 * 30 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 20U; - src = *(in + 32 * 31 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 800; - } -} -void pack_27bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 5U; - src = *(in + 32 * 2 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 15U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 20U; - src = *(in + 32 * 5 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 25U; - src = *(in + 32 * 6 + i); - tmp |= src << 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 3U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 13U; - src = *(in + 32 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 18U; - src = *(in + 32 * 11 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 23U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 1U; - src = *(in + 32 * 14 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 11U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 21U; - src = *(in + 32 * 18 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 26U; - src = *(in + 32 * 19 + i); - tmp |= src << 1U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 9U; - src = *(in + 32 * 22 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 14U; - src = *(in + 32 * 23 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 19U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 3U; - src = *(in + 32 * 26 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 7U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 17U; - src = *(in + 32 * 30 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 22U; - src = *(in + 32 * 31 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 832; - } -} -void pack_28bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 4U; - src = *(in + 32 * 2 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 12U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 20U; - src = *(in + 32 * 6 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 24U; - src = *(in + 32 * 7 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 4U; - src = *(in + 32 * 10 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 12U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 20U; - src = *(in + 32 * 14 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 24U; - src = *(in + 32 * 15 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 4U; - src = *(in + 32 * 18 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 12U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 20U; - src = *(in + 32 * 22 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 24U; - src = *(in + 32 * 23 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 4U; - src = *(in + 32 * 26 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 12U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 20U; - src = *(in + 32 * 30 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 24U; - src = *(in + 32 * 31 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 864; - } -} -void pack_29bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 3U; - src = *(in + 32 * 2 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 9U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 15U; - src = *(in + 32 * 6 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 18U; - src = *(in + 32 * 7 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 21U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 27U; - src = *(in + 32 * 10 + i); - tmp |= src << 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 1U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 7U; - src = *(in + 32 * 14 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 13U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 19U; - src = *(in + 32 * 18 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 22U; - src = *(in + 32 * 19 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 25U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 28U; - src = *(in + 32 * 21 + i); - tmp |= src << 1U; - src = *(in + 32 * 22 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 5U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 11U; - src = *(in + 32 * 26 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 14U; - src = *(in + 32 * 27 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 17U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 23U; - src = *(in + 32 * 30 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 26U; - src = *(in + 32 * 31 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 896; - } -} -void pack_30bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 2U; - src = *(in + 32 * 2 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 6U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 10U; - src = *(in + 32 * 6 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 14U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 18U; - src = *(in + 32 * 10 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 20U; - src = *(in + 32 * 11 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 22U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 24U; - src = *(in + 32 * 13 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 26U; - src = *(in + 32 * 14 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 28U; - src = *(in + 32 * 15 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 2U; - src = *(in + 32 * 18 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 6U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 10U; - src = *(in + 32 * 22 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 14U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 18U; - src = *(in + 32 * 26 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 20U; - src = *(in + 32 * 27 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 22U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 24U; - src = *(in + 32 * 29 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 26U; - src = *(in + 32 * 30 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 28U; - src = *(in + 32 * 31 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 928; - } -} -void pack_31bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 1U; - src = *(in + 32 * 2 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 3U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 5U; - src = *(in + 32 * 6 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 7U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 9U; - src = *(in + 32 * 10 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 11U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 13U; - src = *(in + 32 * 14 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 15U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 17U; - src = *(in + 32 * 18 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 18U; - src = *(in + 32 * 19 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 19U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 21U; - src = *(in + 32 * 22 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 22U; - src = *(in + 32 * 23 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 23U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 25U; - src = *(in + 32 * 26 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 26U; - src = *(in + 32 * 27 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 27U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 28U; - src = *(in + 32 * 29 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 29U; - src = *(in + 32 * 30 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 30U; - src = *(in + 32 * 31 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 960; - } -} -void pack_32bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 31 + i); - tmp = src; - *(out + i) = tmp; - out -= 992; - } -} -void pack_1bit_16simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 1U; - src = *(in + 16 * 2 + i); - tmp |= src << 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 3U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 5U; - src = *(in + 16 * 6 + i); - tmp |= src << 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 7U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 9U; - src = *(in + 16 * 10 + i); - tmp |= src << 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 11U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 13U; - src = *(in + 16 * 14 + i); - tmp |= src << 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 15U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 17U; - src = *(in + 16 * 18 + i); - tmp |= src << 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 19U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 21U; - src = *(in + 16 * 22 + i); - tmp |= src << 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 23U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 25U; - src = *(in + 16 * 26 + i); - tmp |= src << 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 27U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 29U; - src = *(in + 16 * 30 + i); - tmp |= src << 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 31U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 33U; - src = *(in + 16 * 34 + i); - tmp |= src << 34U; - src = *(in + 16 * 35 + i); - tmp |= src << 35U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 37U; - src = *(in + 16 * 38 + i); - tmp |= src << 38U; - src = *(in + 16 * 39 + i); - tmp |= src << 39U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 41U; - src = *(in + 16 * 42 + i); - tmp |= src << 42U; - src = *(in + 16 * 43 + i); - tmp |= src << 43U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 45U; - src = *(in + 16 * 46 + i); - tmp |= src << 46U; - src = *(in + 16 * 47 + i); - tmp |= src << 47U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 49U; - src = *(in + 16 * 50 + i); - tmp |= src << 50U; - src = *(in + 16 * 51 + i); - tmp |= src << 51U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - src = *(in + 16 * 53 + i); - tmp |= src << 53U; - src = *(in + 16 * 54 + i); - tmp |= src << 54U; - src = *(in + 16 * 55 + i); - tmp |= src << 55U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - src = *(in + 16 * 57 + i); - tmp |= src << 57U; - src = *(in + 16 * 58 + i); - tmp |= src << 58U; - src = *(in + 16 * 59 + i); - tmp |= src << 59U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - src = *(in + 16 * 61 + i); - tmp |= src << 61U; - src = *(in + 16 * 62 + i); - tmp |= src << 62U; - src = *(in + 16 * 63 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 2U; - src = *(in + 16 * 2 + i); - tmp |= src << 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 6U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 10U; - src = *(in + 16 * 6 + i); - tmp |= src << 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 14U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 18U; - src = *(in + 16 * 10 + i); - tmp |= src << 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 22U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 26U; - src = *(in + 16 * 14 + i); - tmp |= src << 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 30U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 34U; - src = *(in + 16 * 18 + i); - tmp |= src << 36U; - src = *(in + 16 * 19 + i); - tmp |= src << 38U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 42U; - src = *(in + 16 * 22 + i); - tmp |= src << 44U; - src = *(in + 16 * 23 + i); - tmp |= src << 46U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 50U; - src = *(in + 16 * 26 + i); - tmp |= src << 52U; - src = *(in + 16 * 27 + i); - tmp |= src << 54U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - src = *(in + 16 * 29 + i); - tmp |= src << 58U; - src = *(in + 16 * 30 + i); - tmp |= src << 60U; - src = *(in + 16 * 31 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 2U; - src = *(in + 16 * 34 + i); - tmp |= src << 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 6U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 10U; - src = *(in + 16 * 38 + i); - tmp |= src << 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 14U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 18U; - src = *(in + 16 * 42 + i); - tmp |= src << 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 22U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 26U; - src = *(in + 16 * 46 + i); - tmp |= src << 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 30U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 34U; - src = *(in + 16 * 50 + i); - tmp |= src << 36U; - src = *(in + 16 * 51 + i); - tmp |= src << 38U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 42U; - src = *(in + 16 * 54 + i); - tmp |= src << 44U; - src = *(in + 16 * 55 + i); - tmp |= src << 46U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 50U; - src = *(in + 16 * 58 + i); - tmp |= src << 52U; - src = *(in + 16 * 59 + i); - tmp |= src << 54U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - src = *(in + 16 * 61 + i); - tmp |= src << 58U; - src = *(in + 16 * 62 + i); - tmp |= src << 60U; - src = *(in + 16 * 63 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out -= 16; - } -} -void pack_3bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 3U; - src = *(in + 16 * 2 + i); - tmp |= src << 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 9U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 15U; - src = *(in + 16 * 6 + i); - tmp |= src << 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 21U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 27U; - src = *(in + 16 * 10 + i); - tmp |= src << 30U; - src = *(in + 16 * 11 + i); - tmp |= src << 33U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 39U; - src = *(in + 16 * 14 + i); - tmp |= src << 42U; - src = *(in + 16 * 15 + i); - tmp |= src << 45U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 51U; - src = *(in + 16 * 18 + i); - tmp |= src << 54U; - src = *(in + 16 * 19 + i); - tmp |= src << 57U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - src = *(in + 16 * 21 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 1U; - src = *(in + 16 * 22 + i); - tmp |= src << 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 5U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 11U; - src = *(in + 16 * 26 + i); - tmp |= src << 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 17U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 23U; - src = *(in + 16 * 30 + i); - tmp |= src << 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 29U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 35U; - src = *(in + 16 * 34 + i); - tmp |= src << 38U; - src = *(in + 16 * 35 + i); - tmp |= src << 41U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 47U; - src = *(in + 16 * 38 + i); - tmp |= src << 50U; - src = *(in + 16 * 39 + i); - tmp |= src << 53U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - src = *(in + 16 * 41 + i); - tmp |= src << 59U; - src = *(in + 16 * 42 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 1U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 7U; - src = *(in + 16 * 46 + i); - tmp |= src << 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 13U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 19U; - src = *(in + 16 * 50 + i); - tmp |= src << 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 25U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 31U; - src = *(in + 16 * 54 + i); - tmp |= src << 34U; - src = *(in + 16 * 55 + i); - tmp |= src << 37U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 43U; - src = *(in + 16 * 58 + i); - tmp |= src << 46U; - src = *(in + 16 * 59 + i); - tmp |= src << 49U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - src = *(in + 16 * 61 + i); - tmp |= src << 55U; - src = *(in + 16 * 62 + i); - tmp |= src << 58U; - src = *(in + 16 * 63 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out -= 32; - } -} -void pack_4bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 4U; - src = *(in + 16 * 2 + i); - tmp |= src << 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 12U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 20U; - src = *(in + 16 * 6 + i); - tmp |= src << 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 28U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 36U; - src = *(in + 16 * 10 + i); - tmp |= src << 40U; - src = *(in + 16 * 11 + i); - tmp |= src << 44U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - src = *(in + 16 * 13 + i); - tmp |= src << 52U; - src = *(in + 16 * 14 + i); - tmp |= src << 56U; - src = *(in + 16 * 15 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 4U; - src = *(in + 16 * 18 + i); - tmp |= src << 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 12U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 20U; - src = *(in + 16 * 22 + i); - tmp |= src << 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 28U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 36U; - src = *(in + 16 * 26 + i); - tmp |= src << 40U; - src = *(in + 16 * 27 + i); - tmp |= src << 44U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - src = *(in + 16 * 29 + i); - tmp |= src << 52U; - src = *(in + 16 * 30 + i); - tmp |= src << 56U; - src = *(in + 16 * 31 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 4U; - src = *(in + 16 * 34 + i); - tmp |= src << 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 12U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 20U; - src = *(in + 16 * 38 + i); - tmp |= src << 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 28U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 36U; - src = *(in + 16 * 42 + i); - tmp |= src << 40U; - src = *(in + 16 * 43 + i); - tmp |= src << 44U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - src = *(in + 16 * 45 + i); - tmp |= src << 52U; - src = *(in + 16 * 46 + i); - tmp |= src << 56U; - src = *(in + 16 * 47 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 4U; - src = *(in + 16 * 50 + i); - tmp |= src << 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 12U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 20U; - src = *(in + 16 * 54 + i); - tmp |= src << 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 28U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 36U; - src = *(in + 16 * 58 + i); - tmp |= src << 40U; - src = *(in + 16 * 59 + i); - tmp |= src << 44U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - src = *(in + 16 * 61 + i); - tmp |= src << 52U; - src = *(in + 16 * 62 + i); - tmp |= src << 56U; - src = *(in + 16 * 63 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out -= 48; - } -} -void pack_5bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 5U; - src = *(in + 16 * 2 + i); - tmp |= src << 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 15U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 25U; - src = *(in + 16 * 6 + i); - tmp |= src << 30U; - src = *(in + 16 * 7 + i); - tmp |= src << 35U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 45U; - src = *(in + 16 * 10 + i); - tmp |= src << 50U; - src = *(in + 16 * 11 + i); - tmp |= src << 55U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 1U; - src = *(in + 16 * 14 + i); - tmp |= src << 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 11U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 21U; - src = *(in + 16 * 18 + i); - tmp |= src << 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 31U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 41U; - src = *(in + 16 * 22 + i); - tmp |= src << 46U; - src = *(in + 16 * 23 + i); - tmp |= src << 51U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - src = *(in + 16 * 25 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 3U; - src = *(in + 16 * 26 + i); - tmp |= src << 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 7U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 17U; - src = *(in + 16 * 30 + i); - tmp |= src << 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 27U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 37U; - src = *(in + 16 * 34 + i); - tmp |= src << 42U; - src = *(in + 16 * 35 + i); - tmp |= src << 47U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - src = *(in + 16 * 37 + i); - tmp |= src << 57U; - src = *(in + 16 * 38 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 3U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 13U; - src = *(in + 16 * 42 + i); - tmp |= src << 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 23U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 33U; - src = *(in + 16 * 46 + i); - tmp |= src << 38U; - src = *(in + 16 * 47 + i); - tmp |= src << 43U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 53U; - src = *(in + 16 * 50 + i); - tmp |= src << 58U; - src = *(in + 16 * 51 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 1U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 9U; - src = *(in + 16 * 54 + i); - tmp |= src << 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 19U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 29U; - src = *(in + 16 * 58 + i); - tmp |= src << 34U; - src = *(in + 16 * 59 + i); - tmp |= src << 39U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - src = *(in + 16 * 61 + i); - tmp |= src << 49U; - src = *(in + 16 * 62 + i); - tmp |= src << 54U; - src = *(in + 16 * 63 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out -= 64; - } -} -void pack_6bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 6U; - src = *(in + 16 * 2 + i); - tmp |= src << 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 18U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 30U; - src = *(in + 16 * 6 + i); - tmp |= src << 36U; - src = *(in + 16 * 7 + i); - tmp |= src << 42U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 54U; - src = *(in + 16 * 10 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 2U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 14U; - src = *(in + 16 * 14 + i); - tmp |= src << 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 26U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 38U; - src = *(in + 16 * 18 + i); - tmp |= src << 44U; - src = *(in + 16 * 19 + i); - tmp |= src << 50U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - src = *(in + 16 * 21 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 2U; - src = *(in + 16 * 22 + i); - tmp |= src << 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 10U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 22U; - src = *(in + 16 * 26 + i); - tmp |= src << 28U; - src = *(in + 16 * 27 + i); - tmp |= src << 34U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 46U; - src = *(in + 16 * 30 + i); - tmp |= src << 52U; - src = *(in + 16 * 31 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 6U; - src = *(in + 16 * 34 + i); - tmp |= src << 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 18U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 30U; - src = *(in + 16 * 38 + i); - tmp |= src << 36U; - src = *(in + 16 * 39 + i); - tmp |= src << 42U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 54U; - src = *(in + 16 * 42 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 2U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 14U; - src = *(in + 16 * 46 + i); - tmp |= src << 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 26U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 38U; - src = *(in + 16 * 50 + i); - tmp |= src << 44U; - src = *(in + 16 * 51 + i); - tmp |= src << 50U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - src = *(in + 16 * 53 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 2U; - src = *(in + 16 * 54 + i); - tmp |= src << 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 10U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 22U; - src = *(in + 16 * 58 + i); - tmp |= src << 28U; - src = *(in + 16 * 59 + i); - tmp |= src << 34U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 46U; - src = *(in + 16 * 62 + i); - tmp |= src << 52U; - src = *(in + 16 * 63 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out -= 80; - } -} -void pack_7bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 7U; - src = *(in + 16 * 2 + i); - tmp |= src << 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 21U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 35U; - src = *(in + 16 * 6 + i); - tmp |= src << 42U; - src = *(in + 16 * 7 + i); - tmp |= src << 49U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - src = *(in + 16 * 9 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 1U; - src = *(in + 16 * 10 + i); - tmp |= src << 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 13U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 27U; - src = *(in + 16 * 14 + i); - tmp |= src << 34U; - src = *(in + 16 * 15 + i); - tmp |= src << 41U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 55U; - src = *(in + 16 * 18 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 5U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 19U; - src = *(in + 16 * 22 + i); - tmp |= src << 26U; - src = *(in + 16 * 23 + i); - tmp |= src << 33U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 47U; - src = *(in + 16 * 26 + i); - tmp |= src << 54U; - src = *(in + 16 * 27 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 3U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 11U; - src = *(in + 16 * 30 + i); - tmp |= src << 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 25U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 39U; - src = *(in + 16 * 34 + i); - tmp |= src << 46U; - src = *(in + 16 * 35 + i); - tmp |= src << 53U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 3U; - src = *(in + 16 * 38 + i); - tmp |= src << 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 17U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 31U; - src = *(in + 16 * 42 + i); - tmp |= src << 38U; - src = *(in + 16 * 43 + i); - tmp |= src << 45U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - src = *(in + 16 * 45 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 5U; - src = *(in + 16 * 46 + i); - tmp |= src << 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 9U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 23U; - src = *(in + 16 * 50 + i); - tmp |= src << 30U; - src = *(in + 16 * 51 + i); - tmp |= src << 37U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - src = *(in + 16 * 53 + i); - tmp |= src << 51U; - src = *(in + 16 * 54 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 1U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 15U; - src = *(in + 16 * 58 + i); - tmp |= src << 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 29U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 43U; - src = *(in + 16 * 62 + i); - tmp |= src << 50U; - src = *(in + 16 * 63 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out -= 96; - } -} -void pack_8bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 8U; - src = *(in + 16 * 2 + i); - tmp |= src << 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 24U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 40U; - src = *(in + 16 * 6 + i); - tmp |= src << 48U; - src = *(in + 16 * 7 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 8U; - src = *(in + 16 * 10 + i); - tmp |= src << 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 24U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 40U; - src = *(in + 16 * 14 + i); - tmp |= src << 48U; - src = *(in + 16 * 15 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 8U; - src = *(in + 16 * 18 + i); - tmp |= src << 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 24U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 40U; - src = *(in + 16 * 22 + i); - tmp |= src << 48U; - src = *(in + 16 * 23 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 8U; - src = *(in + 16 * 26 + i); - tmp |= src << 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 24U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 40U; - src = *(in + 16 * 30 + i); - tmp |= src << 48U; - src = *(in + 16 * 31 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 8U; - src = *(in + 16 * 34 + i); - tmp |= src << 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 24U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 40U; - src = *(in + 16 * 38 + i); - tmp |= src << 48U; - src = *(in + 16 * 39 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 8U; - src = *(in + 16 * 42 + i); - tmp |= src << 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 24U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 40U; - src = *(in + 16 * 46 + i); - tmp |= src << 48U; - src = *(in + 16 * 47 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 8U; - src = *(in + 16 * 50 + i); - tmp |= src << 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 24U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 40U; - src = *(in + 16 * 54 + i); - tmp |= src << 48U; - src = *(in + 16 * 55 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 8U; - src = *(in + 16 * 58 + i); - tmp |= src << 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 24U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 40U; - src = *(in + 16 * 62 + i); - tmp |= src << 48U; - src = *(in + 16 * 63 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out -= 112; - } -} -void pack_9bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 9U; - src = *(in + 16 * 2 + i); - tmp |= src << 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 27U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 45U; - src = *(in + 16 * 6 + i); - tmp |= src << 54U; - src = *(in + 16 * 7 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 1U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 17U; - src = *(in + 16 * 10 + i); - tmp |= src << 26U; - src = *(in + 16 * 11 + i); - tmp |= src << 35U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - src = *(in + 16 * 13 + i); - tmp |= src << 53U; - src = *(in + 16 * 14 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 7U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 25U; - src = *(in + 16 * 18 + i); - tmp |= src << 34U; - src = *(in + 16 * 19 + i); - tmp |= src << 43U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - src = *(in + 16 * 21 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 3U; - src = *(in + 16 * 22 + i); - tmp |= src << 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 15U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 33U; - src = *(in + 16 * 26 + i); - tmp |= src << 42U; - src = *(in + 16 * 27 + i); - tmp |= src << 51U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 5U; - src = *(in + 16 * 30 + i); - tmp |= src << 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 23U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 41U; - src = *(in + 16 * 34 + i); - tmp |= src << 50U; - src = *(in + 16 * 35 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 5U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 13U; - src = *(in + 16 * 38 + i); - tmp |= src << 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 31U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 49U; - src = *(in + 16 * 42 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 3U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 21U; - src = *(in + 16 * 46 + i); - tmp |= src << 30U; - src = *(in + 16 * 47 + i); - tmp |= src << 39U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 7U; - src = *(in + 16 * 50 + i); - tmp |= src << 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 11U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 29U; - src = *(in + 16 * 54 + i); - tmp |= src << 38U; - src = *(in + 16 * 55 + i); - tmp |= src << 47U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 1U; - src = *(in + 16 * 58 + i); - tmp |= src << 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 19U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 37U; - src = *(in + 16 * 62 + i); - tmp |= src << 46U; - src = *(in + 16 * 63 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_10bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 10U; - src = *(in + 16 * 2 + i); - tmp |= src << 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 30U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - src = *(in + 16 * 5 + i); - tmp |= src << 50U; - src = *(in + 16 * 6 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 6U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 26U; - src = *(in + 16 * 10 + i); - tmp |= src << 36U; - src = *(in + 16 * 11 + i); - tmp |= src << 46U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 2U; - src = *(in + 16 * 14 + i); - tmp |= src << 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 22U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 42U; - src = *(in + 16 * 18 + i); - tmp |= src << 52U; - src = *(in + 16 * 19 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 2U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 18U; - src = *(in + 16 * 22 + i); - tmp |= src << 28U; - src = *(in + 16 * 23 + i); - tmp |= src << 38U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 6U; - src = *(in + 16 * 26 + i); - tmp |= src << 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 14U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 34U; - src = *(in + 16 * 30 + i); - tmp |= src << 44U; - src = *(in + 16 * 31 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 10U; - src = *(in + 16 * 34 + i); - tmp |= src << 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 30U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - src = *(in + 16 * 37 + i); - tmp |= src << 50U; - src = *(in + 16 * 38 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 6U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 26U; - src = *(in + 16 * 42 + i); - tmp |= src << 36U; - src = *(in + 16 * 43 + i); - tmp |= src << 46U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 2U; - src = *(in + 16 * 46 + i); - tmp |= src << 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 22U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 42U; - src = *(in + 16 * 50 + i); - tmp |= src << 52U; - src = *(in + 16 * 51 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 2U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 18U; - src = *(in + 16 * 54 + i); - tmp |= src << 28U; - src = *(in + 16 * 55 + i); - tmp |= src << 38U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 6U; - src = *(in + 16 * 58 + i); - tmp |= src << 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 14U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 34U; - src = *(in + 16 * 62 + i); - tmp |= src << 44U; - src = *(in + 16 * 63 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out -= 144; - } -} -void pack_11bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 11U; - src = *(in + 16 * 2 + i); - tmp |= src << 22U; - src = *(in + 16 * 3 + i); - tmp |= src << 33U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - src = *(in + 16 * 5 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 9U; - src = *(in + 16 * 6 + i); - tmp |= src << 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 13U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 35U; - src = *(in + 16 * 10 + i); - tmp |= src << 46U; - src = *(in + 16 * 11 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 7U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 15U; - src = *(in + 16 * 14 + i); - tmp |= src << 26U; - src = *(in + 16 * 15 + i); - tmp |= src << 37U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 5U; - src = *(in + 16 * 18 + i); - tmp |= src << 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 17U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 39U; - src = *(in + 16 * 22 + i); - tmp |= src << 50U; - src = *(in + 16 * 23 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 3U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 19U; - src = *(in + 16 * 26 + i); - tmp |= src << 30U; - src = *(in + 16 * 27 + i); - tmp |= src << 41U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - src = *(in + 16 * 29 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 1U; - src = *(in + 16 * 30 + i); - tmp |= src << 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 21U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 43U; - src = *(in + 16 * 34 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 1U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 23U; - src = *(in + 16 * 38 + i); - tmp |= src << 34U; - src = *(in + 16 * 39 + i); - tmp |= src << 45U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 3U; - src = *(in + 16 * 42 + i); - tmp |= src << 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 25U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 47U; - src = *(in + 16 * 46 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 5U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 27U; - src = *(in + 16 * 50 + i); - tmp |= src << 38U; - src = *(in + 16 * 51 + i); - tmp |= src << 49U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 7U; - src = *(in + 16 * 54 + i); - tmp |= src << 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 29U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 51U; - src = *(in + 16 * 58 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 9U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 31U; - src = *(in + 16 * 62 + i); - tmp |= src << 42U; - src = *(in + 16 * 63 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out -= 160; - } -} -void pack_12bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 12U; - src = *(in + 16 * 2 + i); - tmp |= src << 24U; - src = *(in + 16 * 3 + i); - tmp |= src << 36U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - src = *(in + 16 * 5 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 4U; - src = *(in + 16 * 6 + i); - tmp |= src << 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 20U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 44U; - src = *(in + 16 * 10 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 4U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 28U; - src = *(in + 16 * 14 + i); - tmp |= src << 40U; - src = *(in + 16 * 15 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 12U; - src = *(in + 16 * 18 + i); - tmp |= src << 24U; - src = *(in + 16 * 19 + i); - tmp |= src << 36U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - src = *(in + 16 * 21 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 4U; - src = *(in + 16 * 22 + i); - tmp |= src << 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 20U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 44U; - src = *(in + 16 * 26 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 4U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 28U; - src = *(in + 16 * 30 + i); - tmp |= src << 40U; - src = *(in + 16 * 31 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 12U; - src = *(in + 16 * 34 + i); - tmp |= src << 24U; - src = *(in + 16 * 35 + i); - tmp |= src << 36U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - src = *(in + 16 * 37 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 4U; - src = *(in + 16 * 38 + i); - tmp |= src << 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 20U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 44U; - src = *(in + 16 * 42 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 4U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 28U; - src = *(in + 16 * 46 + i); - tmp |= src << 40U; - src = *(in + 16 * 47 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 12U; - src = *(in + 16 * 50 + i); - tmp |= src << 24U; - src = *(in + 16 * 51 + i); - tmp |= src << 36U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - src = *(in + 16 * 53 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 4U; - src = *(in + 16 * 54 + i); - tmp |= src << 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 20U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 44U; - src = *(in + 16 * 58 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 4U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 28U; - src = *(in + 16 * 62 + i); - tmp |= src << 40U; - src = *(in + 16 * 63 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out -= 176; - } -} -void pack_13bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 13U; - src = *(in + 16 * 2 + i); - tmp |= src << 26U; - src = *(in + 16 * 3 + i); - tmp |= src << 39U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 1U; - src = *(in + 16 * 6 + i); - tmp |= src << 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 27U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 11U; - src = *(in + 16 * 10 + i); - tmp |= src << 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 15U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 41U; - src = *(in + 16 * 14 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 3U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 29U; - src = *(in + 16 * 18 + i); - tmp |= src << 42U; - src = *(in + 16 * 19 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 9U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 17U; - src = *(in + 16 * 22 + i); - tmp |= src << 30U; - src = *(in + 16 * 23 + i); - tmp |= src << 43U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 5U; - src = *(in + 16 * 26 + i); - tmp |= src << 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 31U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - src = *(in + 16 * 29 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 7U; - src = *(in + 16 * 30 + i); - tmp |= src << 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 19U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 45U; - src = *(in + 16 * 34 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 7U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 33U; - src = *(in + 16 * 38 + i); - tmp |= src << 46U; - src = *(in + 16 * 39 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 5U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 21U; - src = *(in + 16 * 42 + i); - tmp |= src << 34U; - src = *(in + 16 * 43 + i); - tmp |= src << 47U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 9U; - src = *(in + 16 * 46 + i); - tmp |= src << 22U; - src = *(in + 16 * 47 + i); - tmp |= src << 35U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 3U; - src = *(in + 16 * 50 + i); - tmp |= src << 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 23U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - src = *(in + 16 * 53 + i); - tmp |= src << 49U; - src = *(in + 16 * 54 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 11U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 37U; - src = *(in + 16 * 58 + i); - tmp |= src << 50U; - src = *(in + 16 * 59 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 1U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 25U; - src = *(in + 16 * 62 + i); - tmp |= src << 38U; - src = *(in + 16 * 63 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out -= 192; - } -} -void pack_14bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 14U; - src = *(in + 16 * 2 + i); - tmp |= src << 28U; - src = *(in + 16 * 3 + i); - tmp |= src << 42U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 6U; - src = *(in + 16 * 6 + i); - tmp |= src << 20U; - src = *(in + 16 * 7 + i); - tmp |= src << 34U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 2U; - src = *(in + 16 * 10 + i); - tmp |= src << 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 26U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - src = *(in + 16 * 13 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 10U; - src = *(in + 16 * 14 + i); - tmp |= src << 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 18U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 46U; - src = *(in + 16 * 18 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 10U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 38U; - src = *(in + 16 * 22 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 2U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 30U; - src = *(in + 16 * 26 + i); - tmp |= src << 44U; - src = *(in + 16 * 27 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 6U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 22U; - src = *(in + 16 * 30 + i); - tmp |= src << 36U; - src = *(in + 16 * 31 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 14U; - src = *(in + 16 * 34 + i); - tmp |= src << 28U; - src = *(in + 16 * 35 + i); - tmp |= src << 42U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 6U; - src = *(in + 16 * 38 + i); - tmp |= src << 20U; - src = *(in + 16 * 39 + i); - tmp |= src << 34U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 2U; - src = *(in + 16 * 42 + i); - tmp |= src << 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 26U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - src = *(in + 16 * 45 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 10U; - src = *(in + 16 * 46 + i); - tmp |= src << 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 18U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 46U; - src = *(in + 16 * 50 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 10U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 38U; - src = *(in + 16 * 54 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 2U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 30U; - src = *(in + 16 * 58 + i); - tmp |= src << 44U; - src = *(in + 16 * 59 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 6U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 22U; - src = *(in + 16 * 62 + i); - tmp |= src << 36U; - src = *(in + 16 * 63 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out -= 208; - } -} -void pack_15bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 15U; - src = *(in + 16 * 2 + i); - tmp |= src << 30U; - src = *(in + 16 * 3 + i); - tmp |= src << 45U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 11U; - src = *(in + 16 * 6 + i); - tmp |= src << 26U; - src = *(in + 16 * 7 + i); - tmp |= src << 41U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 7U; - src = *(in + 16 * 10 + i); - tmp |= src << 22U; - src = *(in + 16 * 11 + i); - tmp |= src << 37U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 3U; - src = *(in + 16 * 14 + i); - tmp |= src << 18U; - src = *(in + 16 * 15 + i); - tmp |= src << 33U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 1U; - src = *(in + 16 * 18 + i); - tmp |= src << 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 29U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - src = *(in + 16 * 21 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 5U; - src = *(in + 16 * 22 + i); - tmp |= src << 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 25U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 9U; - src = *(in + 16 * 26 + i); - tmp |= src << 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 21U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - src = *(in + 16 * 29 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 13U; - src = *(in + 16 * 30 + i); - tmp |= src << 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 17U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 47U; - src = *(in + 16 * 34 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 13U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 43U; - src = *(in + 16 * 38 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 9U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 39U; - src = *(in + 16 * 42 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 5U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 35U; - src = *(in + 16 * 46 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 1U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 31U; - src = *(in + 16 * 50 + i); - tmp |= src << 46U; - src = *(in + 16 * 51 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 3U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 27U; - src = *(in + 16 * 54 + i); - tmp |= src << 42U; - src = *(in + 16 * 55 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 7U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 23U; - src = *(in + 16 * 58 + i); - tmp |= src << 38U; - src = *(in + 16 * 59 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 11U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 19U; - src = *(in + 16 * 62 + i); - tmp |= src << 34U; - src = *(in + 16 * 63 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out -= 224; - } -} -void pack_16bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 16U; - src = *(in + 16 * 2 + i); - tmp |= src << 32U; - src = *(in + 16 * 3 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - src = *(in + 16 * 5 + i); - tmp |= src << 16U; - src = *(in + 16 * 6 + i); - tmp |= src << 32U; - src = *(in + 16 * 7 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 16U; - src = *(in + 16 * 10 + i); - tmp |= src << 32U; - src = *(in + 16 * 11 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - src = *(in + 16 * 13 + i); - tmp |= src << 16U; - src = *(in + 16 * 14 + i); - tmp |= src << 32U; - src = *(in + 16 * 15 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 16U; - src = *(in + 16 * 18 + i); - tmp |= src << 32U; - src = *(in + 16 * 19 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - src = *(in + 16 * 21 + i); - tmp |= src << 16U; - src = *(in + 16 * 22 + i); - tmp |= src << 32U; - src = *(in + 16 * 23 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 16U; - src = *(in + 16 * 26 + i); - tmp |= src << 32U; - src = *(in + 16 * 27 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - src = *(in + 16 * 29 + i); - tmp |= src << 16U; - src = *(in + 16 * 30 + i); - tmp |= src << 32U; - src = *(in + 16 * 31 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 16U; - src = *(in + 16 * 34 + i); - tmp |= src << 32U; - src = *(in + 16 * 35 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - src = *(in + 16 * 37 + i); - tmp |= src << 16U; - src = *(in + 16 * 38 + i); - tmp |= src << 32U; - src = *(in + 16 * 39 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 16U; - src = *(in + 16 * 42 + i); - tmp |= src << 32U; - src = *(in + 16 * 43 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - src = *(in + 16 * 45 + i); - tmp |= src << 16U; - src = *(in + 16 * 46 + i); - tmp |= src << 32U; - src = *(in + 16 * 47 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 16U; - src = *(in + 16 * 50 + i); - tmp |= src << 32U; - src = *(in + 16 * 51 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - src = *(in + 16 * 53 + i); - tmp |= src << 16U; - src = *(in + 16 * 54 + i); - tmp |= src << 32U; - src = *(in + 16 * 55 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 16U; - src = *(in + 16 * 58 + i); - tmp |= src << 32U; - src = *(in + 16 * 59 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - src = *(in + 16 * 61 + i); - tmp |= src << 16U; - src = *(in + 16 * 62 + i); - tmp |= src << 32U; - src = *(in + 16 * 63 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out -= 240; - } -} -void pack_17bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 17U; - src = *(in + 16 * 2 + i); - tmp |= src << 34U; - src = *(in + 16 * 3 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 13U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 21U; - src = *(in + 16 * 6 + i); - tmp |= src << 38U; - src = *(in + 16 * 7 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 9U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 25U; - src = *(in + 16 * 10 + i); - tmp |= src << 42U; - src = *(in + 16 * 11 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 5U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 29U; - src = *(in + 16 * 14 + i); - tmp |= src << 46U; - src = *(in + 16 * 15 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 1U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 33U; - src = *(in + 16 * 18 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 3U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 37U; - src = *(in + 16 * 22 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 7U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 41U; - src = *(in + 16 * 26 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 11U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 45U; - src = *(in + 16 * 30 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 15U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 15U; - src = *(in + 16 * 34 + i); - tmp |= src << 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 19U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 11U; - src = *(in + 16 * 38 + i); - tmp |= src << 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 23U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 7U; - src = *(in + 16 * 42 + i); - tmp |= src << 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 27U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 3U; - src = *(in + 16 * 46 + i); - tmp |= src << 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 31U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 1U; - src = *(in + 16 * 50 + i); - tmp |= src << 18U; - src = *(in + 16 * 51 + i); - tmp |= src << 35U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 5U; - src = *(in + 16 * 54 + i); - tmp |= src << 22U; - src = *(in + 16 * 55 + i); - tmp |= src << 39U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 9U; - src = *(in + 16 * 58 + i); - tmp |= src << 26U; - src = *(in + 16 * 59 + i); - tmp |= src << 43U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 13U; - src = *(in + 16 * 62 + i); - tmp |= src << 30U; - src = *(in + 16 * 63 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_18bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 18U; - src = *(in + 16 * 2 + i); - tmp |= src << 36U; - src = *(in + 16 * 3 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 10U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 26U; - src = *(in + 16 * 6 + i); - tmp |= src << 44U; - src = *(in + 16 * 7 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 2U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 34U; - src = *(in + 16 * 10 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 6U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 42U; - src = *(in + 16 * 14 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 14U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 14U; - src = *(in + 16 * 18 + i); - tmp |= src << 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 22U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 6U; - src = *(in + 16 * 22 + i); - tmp |= src << 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 30U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 2U; - src = *(in + 16 * 26 + i); - tmp |= src << 20U; - src = *(in + 16 * 27 + i); - tmp |= src << 38U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 10U; - src = *(in + 16 * 30 + i); - tmp |= src << 28U; - src = *(in + 16 * 31 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 18U; - src = *(in + 16 * 34 + i); - tmp |= src << 36U; - src = *(in + 16 * 35 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 10U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 26U; - src = *(in + 16 * 38 + i); - tmp |= src << 44U; - src = *(in + 16 * 39 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 2U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 34U; - src = *(in + 16 * 42 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 6U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 42U; - src = *(in + 16 * 46 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 14U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 14U; - src = *(in + 16 * 50 + i); - tmp |= src << 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 22U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 6U; - src = *(in + 16 * 54 + i); - tmp |= src << 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 30U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 2U; - src = *(in + 16 * 58 + i); - tmp |= src << 20U; - src = *(in + 16 * 59 + i); - tmp |= src << 38U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 10U; - src = *(in + 16 * 62 + i); - tmp |= src << 28U; - src = *(in + 16 * 63 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out -= 272; - } -} -void pack_19bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 19U; - src = *(in + 16 * 2 + i); - tmp |= src << 38U; - src = *(in + 16 * 3 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 7U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 31U; - src = *(in + 16 * 6 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 5U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 43U; - src = *(in + 16 * 10 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 17U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 9U; - src = *(in + 16 * 14 + i); - tmp |= src << 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 29U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 3U; - src = *(in + 16 * 18 + i); - tmp |= src << 22U; - src = *(in + 16 * 19 + i); - tmp |= src << 41U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 15U; - src = *(in + 16 * 22 + i); - tmp |= src << 34U; - src = *(in + 16 * 23 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 11U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 27U; - src = *(in + 16 * 26 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 1U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 39U; - src = *(in + 16 * 30 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 13U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 13U; - src = *(in + 16 * 34 + i); - tmp |= src << 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 25U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 1U; - src = *(in + 16 * 38 + i); - tmp |= src << 18U; - src = *(in + 16 * 39 + i); - tmp |= src << 37U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 11U; - src = *(in + 16 * 42 + i); - tmp |= src << 30U; - src = *(in + 16 * 43 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 15U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 23U; - src = *(in + 16 * 46 + i); - tmp |= src << 42U; - src = *(in + 16 * 47 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 3U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 35U; - src = *(in + 16 * 50 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 9U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 17U; - src = *(in + 16 * 54 + i); - tmp |= src << 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 21U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 5U; - src = *(in + 16 * 58 + i); - tmp |= src << 14U; - src = *(in + 16 * 59 + i); - tmp |= src << 33U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 7U; - src = *(in + 16 * 62 + i); - tmp |= src << 26U; - src = *(in + 16 * 63 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out -= 288; - } -} -void pack_20bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 20U; - src = *(in + 16 * 2 + i); - tmp |= src << 40U; - src = *(in + 16 * 3 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 4U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 36U; - src = *(in + 16 * 6 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 12U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 12U; - src = *(in + 16 * 10 + i); - tmp |= src << 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 28U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 4U; - src = *(in + 16 * 14 + i); - tmp |= src << 24U; - src = *(in + 16 * 15 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 20U; - src = *(in + 16 * 18 + i); - tmp |= src << 40U; - src = *(in + 16 * 19 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 4U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 36U; - src = *(in + 16 * 22 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 12U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 12U; - src = *(in + 16 * 26 + i); - tmp |= src << 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 28U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 4U; - src = *(in + 16 * 30 + i); - tmp |= src << 24U; - src = *(in + 16 * 31 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 20U; - src = *(in + 16 * 34 + i); - tmp |= src << 40U; - src = *(in + 16 * 35 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 4U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 36U; - src = *(in + 16 * 38 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 12U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 12U; - src = *(in + 16 * 42 + i); - tmp |= src << 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 28U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 4U; - src = *(in + 16 * 46 + i); - tmp |= src << 24U; - src = *(in + 16 * 47 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 20U; - src = *(in + 16 * 50 + i); - tmp |= src << 40U; - src = *(in + 16 * 51 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 4U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 36U; - src = *(in + 16 * 54 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 12U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 12U; - src = *(in + 16 * 58 + i); - tmp |= src << 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 28U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 4U; - src = *(in + 16 * 62 + i); - tmp |= src << 24U; - src = *(in + 16 * 63 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out -= 304; - } -} -void pack_21bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 21U; - src = *(in + 16 * 2 + i); - tmp |= src << 42U; - src = *(in + 16 * 3 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 1U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 41U; - src = *(in + 16 * 6 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 19U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 3U; - src = *(in + 16 * 10 + i); - tmp |= src << 18U; - src = *(in + 16 * 11 + i); - tmp |= src << 39U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 17U; - src = *(in + 16 * 14 + i); - tmp |= src << 38U; - src = *(in + 16 * 15 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 5U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 37U; - src = *(in + 16 * 18 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 15U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 7U; - src = *(in + 16 * 22 + i); - tmp |= src << 14U; - src = *(in + 16 * 23 + i); - tmp |= src << 35U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 13U; - src = *(in + 16 * 26 + i); - tmp |= src << 34U; - src = *(in + 16 * 27 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 9U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 33U; - src = *(in + 16 * 30 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 11U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 11U; - src = *(in + 16 * 34 + i); - tmp |= src << 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 31U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 9U; - src = *(in + 16 * 38 + i); - tmp |= src << 30U; - src = *(in + 16 * 39 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 13U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 29U; - src = *(in + 16 * 42 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 7U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 15U; - src = *(in + 16 * 46 + i); - tmp |= src << 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 27U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 5U; - src = *(in + 16 * 50 + i); - tmp |= src << 26U; - src = *(in + 16 * 51 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 17U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 25U; - src = *(in + 16 * 54 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 3U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 19U; - src = *(in + 16 * 58 + i); - tmp |= src << 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 23U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 1U; - src = *(in + 16 * 62 + i); - tmp |= src << 22U; - src = *(in + 16 * 63 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out -= 320; - } -} -void pack_22bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 22U; - src = *(in + 16 * 2 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 2U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 18U; - src = *(in + 16 * 6 + i); - tmp |= src << 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 26U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 6U; - src = *(in + 16 * 10 + i); - tmp |= src << 28U; - src = *(in + 16 * 11 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 14U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 30U; - src = *(in + 16 * 14 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 10U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 10U; - src = *(in + 16 * 18 + i); - tmp |= src << 12U; - src = *(in + 16 * 19 + i); - tmp |= src << 34U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 14U; - src = *(in + 16 * 22 + i); - tmp |= src << 36U; - src = *(in + 16 * 23 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 6U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 38U; - src = *(in + 16 * 26 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 18U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 2U; - src = *(in + 16 * 30 + i); - tmp |= src << 20U; - src = *(in + 16 * 31 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 22U; - src = *(in + 16 * 34 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 2U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 18U; - src = *(in + 16 * 38 + i); - tmp |= src << 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 26U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 6U; - src = *(in + 16 * 42 + i); - tmp |= src << 28U; - src = *(in + 16 * 43 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 14U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 30U; - src = *(in + 16 * 46 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 10U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 10U; - src = *(in + 16 * 50 + i); - tmp |= src << 12U; - src = *(in + 16 * 51 + i); - tmp |= src << 34U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 14U; - src = *(in + 16 * 54 + i); - tmp |= src << 36U; - src = *(in + 16 * 55 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 6U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 38U; - src = *(in + 16 * 58 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 18U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 2U; - src = *(in + 16 * 62 + i); - tmp |= src << 20U; - src = *(in + 16 * 63 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out -= 336; - } -} -void pack_23bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 23U; - src = *(in + 16 * 2 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 5U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 13U; - src = *(in + 16 * 6 + i); - tmp |= src << 10U; - src = *(in + 16 * 7 + i); - tmp |= src << 33U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 15U; - src = *(in + 16 * 10 + i); - tmp |= src << 38U; - src = *(in + 16 * 11 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 3U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 21U; - src = *(in + 16 * 14 + i); - tmp |= src << 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 25U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 7U; - src = *(in + 16 * 18 + i); - tmp |= src << 30U; - src = *(in + 16 * 19 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 11U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 35U; - src = *(in + 16 * 22 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 17U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 1U; - src = *(in + 16 * 26 + i); - tmp |= src << 22U; - src = *(in + 16 * 27 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 19U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 27U; - src = *(in + 16 * 30 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 9U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 9U; - src = *(in + 16 * 34 + i); - tmp |= src << 14U; - src = *(in + 16 * 35 + i); - tmp |= src << 37U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 19U; - src = *(in + 16 * 38 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 1U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 17U; - src = *(in + 16 * 42 + i); - tmp |= src << 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 29U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 11U; - src = *(in + 16 * 46 + i); - tmp |= src << 34U; - src = *(in + 16 * 47 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 7U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 39U; - src = *(in + 16 * 50 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 21U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 3U; - src = *(in + 16 * 54 + i); - tmp |= src << 26U; - src = *(in + 16 * 55 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 15U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 31U; - src = *(in + 16 * 58 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 13U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 5U; - src = *(in + 16 * 62 + i); - tmp |= src << 18U; - src = *(in + 16 * 63 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out -= 352; - } -} -void pack_24bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 24U; - src = *(in + 16 * 2 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 8U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 8U; - src = *(in + 16 * 6 + i); - tmp |= src << 16U; - src = *(in + 16 * 7 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 24U; - src = *(in + 16 * 10 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 8U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 8U; - src = *(in + 16 * 14 + i); - tmp |= src << 16U; - src = *(in + 16 * 15 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 24U; - src = *(in + 16 * 18 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 8U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 8U; - src = *(in + 16 * 22 + i); - tmp |= src << 16U; - src = *(in + 16 * 23 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 24U; - src = *(in + 16 * 26 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 8U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 8U; - src = *(in + 16 * 30 + i); - tmp |= src << 16U; - src = *(in + 16 * 31 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 24U; - src = *(in + 16 * 34 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 8U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 8U; - src = *(in + 16 * 38 + i); - tmp |= src << 16U; - src = *(in + 16 * 39 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 24U; - src = *(in + 16 * 42 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 8U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 8U; - src = *(in + 16 * 46 + i); - tmp |= src << 16U; - src = *(in + 16 * 47 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 24U; - src = *(in + 16 * 50 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 8U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 8U; - src = *(in + 16 * 54 + i); - tmp |= src << 16U; - src = *(in + 16 * 55 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 24U; - src = *(in + 16 * 58 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 8U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 8U; - src = *(in + 16 * 62 + i); - tmp |= src << 16U; - src = *(in + 16 * 63 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out -= 368; - } -} -void pack_25bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 25U; - src = *(in + 16 * 2 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 11U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 3U; - src = *(in + 16 * 6 + i); - tmp |= src << 22U; - src = *(in + 16 * 7 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 17U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 33U; - src = *(in + 16 * 10 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 19U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 5U; - src = *(in + 16 * 14 + i); - tmp |= src << 30U; - src = *(in + 16 * 15 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 9U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 23U; - src = *(in + 16 * 18 + i); - tmp |= src << 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 27U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 13U; - src = *(in + 16 * 22 + i); - tmp |= src << 38U; - src = *(in + 16 * 23 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 1U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 15U; - src = *(in + 16 * 26 + i); - tmp |= src << 10U; - src = *(in + 16 * 27 + i); - tmp |= src << 35U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 21U; - src = *(in + 16 * 30 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 7U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 7U; - src = *(in + 16 * 34 + i); - tmp |= src << 18U; - src = *(in + 16 * 35 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 21U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 29U; - src = *(in + 16 * 38 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 15U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 1U; - src = *(in + 16 * 42 + i); - tmp |= src << 26U; - src = *(in + 16 * 43 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 13U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 37U; - src = *(in + 16 * 46 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 23U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 9U; - src = *(in + 16 * 50 + i); - tmp |= src << 34U; - src = *(in + 16 * 51 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 5U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 19U; - src = *(in + 16 * 54 + i); - tmp |= src << 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 31U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 17U; - src = *(in + 16 * 58 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 3U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 11U; - src = *(in + 16 * 62 + i); - tmp |= src << 14U; - src = *(in + 16 * 63 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_26bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 26U; - src = *(in + 16 * 2 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 14U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 2U; - src = *(in + 16 * 6 + i); - tmp |= src << 28U; - src = *(in + 16 * 7 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 10U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 22U; - src = *(in + 16 * 10 + i); - tmp |= src << 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 30U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 18U; - src = *(in + 16 * 14 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 6U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 6U; - src = *(in + 16 * 18 + i); - tmp |= src << 20U; - src = *(in + 16 * 19 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 18U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 34U; - src = *(in + 16 * 22 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 22U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 10U; - src = *(in + 16 * 26 + i); - tmp |= src << 36U; - src = *(in + 16 * 27 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 2U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 14U; - src = *(in + 16 * 30 + i); - tmp |= src << 12U; - src = *(in + 16 * 31 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 26U; - src = *(in + 16 * 34 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 14U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 2U; - src = *(in + 16 * 38 + i); - tmp |= src << 28U; - src = *(in + 16 * 39 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 10U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 22U; - src = *(in + 16 * 42 + i); - tmp |= src << 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 30U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 18U; - src = *(in + 16 * 46 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 6U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 6U; - src = *(in + 16 * 50 + i); - tmp |= src << 20U; - src = *(in + 16 * 51 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 18U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 34U; - src = *(in + 16 * 54 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 22U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 10U; - src = *(in + 16 * 58 + i); - tmp |= src << 36U; - src = *(in + 16 * 59 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 2U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 14U; - src = *(in + 16 * 62 + i); - tmp |= src << 12U; - src = *(in + 16 * 63 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out -= 400; - } -} -void pack_27bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 27U; - src = *(in + 16 * 2 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 17U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 7U; - src = *(in + 16 * 6 + i); - tmp |= src << 34U; - src = *(in + 16 * 7 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 3U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 13U; - src = *(in + 16 * 10 + i); - tmp |= src << 14U; - src = *(in + 16 * 11 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 23U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 31U; - src = *(in + 16 * 14 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 21U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 11U; - src = *(in + 16 * 18 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 1U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 9U; - src = *(in + 16 * 22 + i); - tmp |= src << 18U; - src = *(in + 16 * 23 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 19U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 35U; - src = *(in + 16 * 26 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 25U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 15U; - src = *(in + 16 * 30 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 5U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 5U; - src = *(in + 16 * 34 + i); - tmp |= src << 22U; - src = *(in + 16 * 35 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 15U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 25U; - src = *(in + 16 * 38 + i); - tmp |= src << 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 29U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 19U; - src = *(in + 16 * 42 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 9U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 1U; - src = *(in + 16 * 46 + i); - tmp |= src << 26U; - src = *(in + 16 * 47 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 11U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 21U; - src = *(in + 16 * 50 + i); - tmp |= src << 6U; - src = *(in + 16 * 51 + i); - tmp |= src << 33U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 23U; - src = *(in + 16 * 54 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 13U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 3U; - src = *(in + 16 * 58 + i); - tmp |= src << 30U; - src = *(in + 16 * 59 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 7U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 17U; - src = *(in + 16 * 62 + i); - tmp |= src << 10U; - src = *(in + 16 * 63 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out -= 416; - } -} -void pack_28bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 28U; - src = *(in + 16 * 2 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 20U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 12U; - src = *(in + 16 * 6 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 4U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 4U; - src = *(in + 16 * 10 + i); - tmp |= src << 24U; - src = *(in + 16 * 11 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 12U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 20U; - src = *(in + 16 * 14 + i); - tmp |= src << 8U; - src = *(in + 16 * 15 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 28U; - src = *(in + 16 * 18 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 20U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 12U; - src = *(in + 16 * 22 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 4U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 4U; - src = *(in + 16 * 26 + i); - tmp |= src << 24U; - src = *(in + 16 * 27 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 12U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 20U; - src = *(in + 16 * 30 + i); - tmp |= src << 8U; - src = *(in + 16 * 31 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 28U; - src = *(in + 16 * 34 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 20U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 12U; - src = *(in + 16 * 38 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 4U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 4U; - src = *(in + 16 * 42 + i); - tmp |= src << 24U; - src = *(in + 16 * 43 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 12U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 20U; - src = *(in + 16 * 46 + i); - tmp |= src << 8U; - src = *(in + 16 * 47 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 28U; - src = *(in + 16 * 50 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 20U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 12U; - src = *(in + 16 * 54 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 4U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 4U; - src = *(in + 16 * 58 + i); - tmp |= src << 24U; - src = *(in + 16 * 59 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 12U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 20U; - src = *(in + 16 * 62 + i); - tmp |= src << 8U; - src = *(in + 16 * 63 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out -= 432; - } -} -void pack_29bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 29U; - src = *(in + 16 * 2 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 23U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 17U; - src = *(in + 16 * 6 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 11U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 5U; - src = *(in + 16 * 10 + i); - tmp |= src << 34U; - src = *(in + 16 * 11 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 1U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 7U; - src = *(in + 16 * 14 + i); - tmp |= src << 22U; - src = *(in + 16 * 15 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 13U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 19U; - src = *(in + 16 * 18 + i); - tmp |= src << 10U; - src = *(in + 16 * 19 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 25U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 33U; - src = *(in + 16 * 22 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 27U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 21U; - src = *(in + 16 * 26 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 15U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 9U; - src = *(in + 16 * 30 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 3U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 3U; - src = *(in + 16 * 34 + i); - tmp |= src << 26U; - src = *(in + 16 * 35 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 9U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 15U; - src = *(in + 16 * 38 + i); - tmp |= src << 14U; - src = *(in + 16 * 39 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 21U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 27U; - src = *(in + 16 * 42 + i); - tmp |= src << 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 31U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 25U; - src = *(in + 16 * 46 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 19U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 13U; - src = *(in + 16 * 50 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 7U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 1U; - src = *(in + 16 * 54 + i); - tmp |= src << 30U; - src = *(in + 16 * 55 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 5U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 11U; - src = *(in + 16 * 58 + i); - tmp |= src << 18U; - src = *(in + 16 * 59 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 17U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 23U; - src = *(in + 16 * 62 + i); - tmp |= src << 6U; - src = *(in + 16 * 63 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out -= 448; - } -} -void pack_30bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 30U; - src = *(in + 16 * 2 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 26U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 22U; - src = *(in + 16 * 6 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 18U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 14U; - src = *(in + 16 * 10 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 10U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 6U; - src = *(in + 16 * 14 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 2U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 2U; - src = *(in + 16 * 18 + i); - tmp |= src << 28U; - src = *(in + 16 * 19 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 6U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 10U; - src = *(in + 16 * 22 + i); - tmp |= src << 20U; - src = *(in + 16 * 23 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 14U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 18U; - src = *(in + 16 * 26 + i); - tmp |= src << 12U; - src = *(in + 16 * 27 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 22U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 26U; - src = *(in + 16 * 30 + i); - tmp |= src << 4U; - src = *(in + 16 * 31 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 30U; - src = *(in + 16 * 34 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 26U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 22U; - src = *(in + 16 * 38 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 18U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 14U; - src = *(in + 16 * 42 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 10U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 6U; - src = *(in + 16 * 46 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 2U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 2U; - src = *(in + 16 * 50 + i); - tmp |= src << 28U; - src = *(in + 16 * 51 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 6U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 10U; - src = *(in + 16 * 54 + i); - tmp |= src << 20U; - src = *(in + 16 * 55 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 14U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 18U; - src = *(in + 16 * 58 + i); - tmp |= src << 12U; - src = *(in + 16 * 59 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 22U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 26U; - src = *(in + 16 * 62 + i); - tmp |= src << 4U; - src = *(in + 16 * 63 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out -= 464; - } -} -void pack_31bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 31U; - src = *(in + 16 * 2 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 29U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 27U; - src = *(in + 16 * 6 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 25U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 23U; - src = *(in + 16 * 10 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 21U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 19U; - src = *(in + 16 * 14 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 17U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 15U; - src = *(in + 16 * 18 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 13U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 11U; - src = *(in + 16 * 22 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 9U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 7U; - src = *(in + 16 * 26 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 5U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 3U; - src = *(in + 16 * 30 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 1U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 1U; - src = *(in + 16 * 34 + i); - tmp |= src << 30U; - src = *(in + 16 * 35 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 3U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 5U; - src = *(in + 16 * 38 + i); - tmp |= src << 26U; - src = *(in + 16 * 39 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 7U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 9U; - src = *(in + 16 * 42 + i); - tmp |= src << 22U; - src = *(in + 16 * 43 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 11U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 13U; - src = *(in + 16 * 46 + i); - tmp |= src << 18U; - src = *(in + 16 * 47 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 15U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 17U; - src = *(in + 16 * 50 + i); - tmp |= src << 14U; - src = *(in + 16 * 51 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 19U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 21U; - src = *(in + 16 * 54 + i); - tmp |= src << 10U; - src = *(in + 16 * 55 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 23U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 25U; - src = *(in + 16 * 58 + i); - tmp |= src << 6U; - src = *(in + 16 * 59 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 27U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 29U; - src = *(in + 16 * 62 + i); - tmp |= src << 2U; - src = *(in + 16 * 63 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out -= 480; - } -} -void pack_32bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src; - src = *(in + 16 * 3 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - src = *(in + 16 * 5 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src; - src = *(in + 16 * 7 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src; - src = *(in + 16 * 11 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - src = *(in + 16 * 13 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src; - src = *(in + 16 * 15 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src; - src = *(in + 16 * 19 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - src = *(in + 16 * 21 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src; - src = *(in + 16 * 23 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src; - src = *(in + 16 * 27 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - src = *(in + 16 * 29 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src; - src = *(in + 16 * 31 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src; - src = *(in + 16 * 35 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - src = *(in + 16 * 37 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src; - src = *(in + 16 * 39 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src; - src = *(in + 16 * 43 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - src = *(in + 16 * 45 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src; - src = *(in + 16 * 47 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src; - src = *(in + 16 * 51 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - src = *(in + 16 * 53 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src; - src = *(in + 16 * 55 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src; - src = *(in + 16 * 59 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - src = *(in + 16 * 61 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src; - src = *(in + 16 * 63 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out -= 496; - } -} -void pack_33bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 31U; - src = *(in + 16 * 2 + i); - tmp |= src << 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 29U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 27U; - src = *(in + 16 * 6 + i); - tmp |= src << 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 25U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 23U; - src = *(in + 16 * 10 + i); - tmp |= src << 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 21U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 19U; - src = *(in + 16 * 14 + i); - tmp |= src << 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 17U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 15U; - src = *(in + 16 * 18 + i); - tmp |= src << 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 13U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 11U; - src = *(in + 16 * 22 + i); - tmp |= src << 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 9U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 7U; - src = *(in + 16 * 26 + i); - tmp |= src << 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 5U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 3U; - src = *(in + 16 * 30 + i); - tmp |= src << 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 1U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 1U; - src = *(in + 16 * 34 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 30U; - src = *(in + 16 * 35 + i); - tmp |= src << 3U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 5U; - src = *(in + 16 * 38 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 26U; - src = *(in + 16 * 39 + i); - tmp |= src << 7U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 9U; - src = *(in + 16 * 42 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 22U; - src = *(in + 16 * 43 + i); - tmp |= src << 11U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 13U; - src = *(in + 16 * 46 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 18U; - src = *(in + 16 * 47 + i); - tmp |= src << 15U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 17U; - src = *(in + 16 * 50 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 14U; - src = *(in + 16 * 51 + i); - tmp |= src << 19U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 21U; - src = *(in + 16 * 54 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 10U; - src = *(in + 16 * 55 + i); - tmp |= src << 23U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 25U; - src = *(in + 16 * 58 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 6U; - src = *(in + 16 * 59 + i); - tmp |= src << 27U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 29U; - src = *(in + 16 * 62 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 2U; - src = *(in + 16 * 63 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_34bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 30U; - src = *(in + 16 * 2 + i); - tmp |= src << 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 26U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 22U; - src = *(in + 16 * 6 + i); - tmp |= src << 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 18U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 14U; - src = *(in + 16 * 10 + i); - tmp |= src << 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 10U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 6U; - src = *(in + 16 * 14 + i); - tmp |= src << 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 2U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 2U; - src = *(in + 16 * 18 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 28U; - src = *(in + 16 * 19 + i); - tmp |= src << 6U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 10U; - src = *(in + 16 * 22 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 20U; - src = *(in + 16 * 23 + i); - tmp |= src << 14U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 18U; - src = *(in + 16 * 26 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 12U; - src = *(in + 16 * 27 + i); - tmp |= src << 22U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 26U; - src = *(in + 16 * 30 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 4U; - src = *(in + 16 * 31 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 30U; - src = *(in + 16 * 34 + i); - tmp |= src << 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 26U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 22U; - src = *(in + 16 * 38 + i); - tmp |= src << 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 18U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 14U; - src = *(in + 16 * 42 + i); - tmp |= src << 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 10U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 6U; - src = *(in + 16 * 46 + i); - tmp |= src << 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 2U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 2U; - src = *(in + 16 * 50 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 28U; - src = *(in + 16 * 51 + i); - tmp |= src << 6U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 10U; - src = *(in + 16 * 54 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 20U; - src = *(in + 16 * 55 + i); - tmp |= src << 14U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 18U; - src = *(in + 16 * 58 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 12U; - src = *(in + 16 * 59 + i); - tmp |= src << 22U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 26U; - src = *(in + 16 * 62 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 4U; - src = *(in + 16 * 63 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out -= 528; - } -} -void pack_35bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 29U; - src = *(in + 16 * 2 + i); - tmp |= src << 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 23U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 17U; - src = *(in + 16 * 6 + i); - tmp |= src << 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 11U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 5U; - src = *(in + 16 * 10 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 34U; - src = *(in + 16 * 11 + i); - tmp |= src << 1U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 7U; - src = *(in + 16 * 14 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 22U; - src = *(in + 16 * 15 + i); - tmp |= src << 13U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 19U; - src = *(in + 16 * 18 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 10U; - src = *(in + 16 * 19 + i); - tmp |= src << 25U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 33U; - src = *(in + 16 * 22 + i); - tmp |= src << 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 27U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 21U; - src = *(in + 16 * 26 + i); - tmp |= src << 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 15U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 9U; - src = *(in + 16 * 30 + i); - tmp |= src << 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 3U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 3U; - src = *(in + 16 * 34 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 26U; - src = *(in + 16 * 35 + i); - tmp |= src << 9U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 15U; - src = *(in + 16 * 38 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 14U; - src = *(in + 16 * 39 + i); - tmp |= src << 21U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 27U; - src = *(in + 16 * 42 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 31U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 25U; - src = *(in + 16 * 46 + i); - tmp |= src << 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 19U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 13U; - src = *(in + 16 * 50 + i); - tmp |= src << 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 7U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 1U; - src = *(in + 16 * 54 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 30U; - src = *(in + 16 * 55 + i); - tmp |= src << 5U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 11U; - src = *(in + 16 * 58 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 18U; - src = *(in + 16 * 59 + i); - tmp |= src << 17U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 23U; - src = *(in + 16 * 62 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 6U; - src = *(in + 16 * 63 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out -= 544; - } -} -void pack_36bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 28U; - src = *(in + 16 * 2 + i); - tmp |= src << 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 20U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 12U; - src = *(in + 16 * 6 + i); - tmp |= src << 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 4U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 4U; - src = *(in + 16 * 10 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 24U; - src = *(in + 16 * 11 + i); - tmp |= src << 12U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 20U; - src = *(in + 16 * 14 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 8U; - src = *(in + 16 * 15 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 28U; - src = *(in + 16 * 18 + i); - tmp |= src << 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 20U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 12U; - src = *(in + 16 * 22 + i); - tmp |= src << 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 4U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 4U; - src = *(in + 16 * 26 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 24U; - src = *(in + 16 * 27 + i); - tmp |= src << 12U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 20U; - src = *(in + 16 * 30 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 8U; - src = *(in + 16 * 31 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 28U; - src = *(in + 16 * 34 + i); - tmp |= src << 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 20U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 12U; - src = *(in + 16 * 38 + i); - tmp |= src << 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 4U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 4U; - src = *(in + 16 * 42 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 24U; - src = *(in + 16 * 43 + i); - tmp |= src << 12U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 20U; - src = *(in + 16 * 46 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 8U; - src = *(in + 16 * 47 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 28U; - src = *(in + 16 * 50 + i); - tmp |= src << 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 20U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 12U; - src = *(in + 16 * 54 + i); - tmp |= src << 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 4U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 4U; - src = *(in + 16 * 58 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 24U; - src = *(in + 16 * 59 + i); - tmp |= src << 12U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 20U; - src = *(in + 16 * 62 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 8U; - src = *(in + 16 * 63 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out -= 560; - } -} -void pack_37bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 27U; - src = *(in + 16 * 2 + i); - tmp |= src << 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 17U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 7U; - src = *(in + 16 * 6 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 34U; - src = *(in + 16 * 7 + i); - tmp |= src << 3U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 13U; - src = *(in + 16 * 10 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 14U; - src = *(in + 16 * 11 + i); - tmp |= src << 23U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 31U; - src = *(in + 16 * 14 + i); - tmp |= src << 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 21U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 11U; - src = *(in + 16 * 18 + i); - tmp |= src << 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 1U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 9U; - src = *(in + 16 * 22 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 18U; - src = *(in + 16 * 23 + i); - tmp |= src << 19U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 35U; - src = *(in + 16 * 26 + i); - tmp |= src << 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 25U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 15U; - src = *(in + 16 * 30 + i); - tmp |= src << 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 5U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 5U; - src = *(in + 16 * 34 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 22U; - src = *(in + 16 * 35 + i); - tmp |= src << 15U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 25U; - src = *(in + 16 * 38 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 29U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 19U; - src = *(in + 16 * 42 + i); - tmp |= src << 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 9U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 1U; - src = *(in + 16 * 46 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 26U; - src = *(in + 16 * 47 + i); - tmp |= src << 11U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 21U; - src = *(in + 16 * 50 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 6U; - src = *(in + 16 * 51 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 33U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 23U; - src = *(in + 16 * 54 + i); - tmp |= src << 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 13U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 3U; - src = *(in + 16 * 58 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 30U; - src = *(in + 16 * 59 + i); - tmp |= src << 7U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 17U; - src = *(in + 16 * 62 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 10U; - src = *(in + 16 * 63 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out -= 576; - } -} -void pack_38bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 26U; - src = *(in + 16 * 2 + i); - tmp |= src << 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 14U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 2U; - src = *(in + 16 * 6 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 28U; - src = *(in + 16 * 7 + i); - tmp |= src << 10U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 22U; - src = *(in + 16 * 10 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 30U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 18U; - src = *(in + 16 * 14 + i); - tmp |= src << 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 6U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 6U; - src = *(in + 16 * 18 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 20U; - src = *(in + 16 * 19 + i); - tmp |= src << 18U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 34U; - src = *(in + 16 * 22 + i); - tmp |= src << 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 22U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 10U; - src = *(in + 16 * 26 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 36U; - src = *(in + 16 * 27 + i); - tmp |= src << 2U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 14U; - src = *(in + 16 * 30 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 12U; - src = *(in + 16 * 31 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 26U; - src = *(in + 16 * 34 + i); - tmp |= src << 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 14U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 2U; - src = *(in + 16 * 38 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 28U; - src = *(in + 16 * 39 + i); - tmp |= src << 10U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 22U; - src = *(in + 16 * 42 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 30U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 18U; - src = *(in + 16 * 46 + i); - tmp |= src << 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 6U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 6U; - src = *(in + 16 * 50 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 20U; - src = *(in + 16 * 51 + i); - tmp |= src << 18U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 34U; - src = *(in + 16 * 54 + i); - tmp |= src << 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 22U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 10U; - src = *(in + 16 * 58 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 36U; - src = *(in + 16 * 59 + i); - tmp |= src << 2U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 14U; - src = *(in + 16 * 62 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 12U; - src = *(in + 16 * 63 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out -= 592; - } -} -void pack_39bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 25U; - src = *(in + 16 * 2 + i); - tmp |= src << 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 11U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 3U; - src = *(in + 16 * 6 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 22U; - src = *(in + 16 * 7 + i); - tmp |= src << 17U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 33U; - src = *(in + 16 * 10 + i); - tmp |= src << 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 19U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 5U; - src = *(in + 16 * 14 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 30U; - src = *(in + 16 * 15 + i); - tmp |= src << 9U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 23U; - src = *(in + 16 * 18 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 27U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 13U; - src = *(in + 16 * 22 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 38U; - src = *(in + 16 * 23 + i); - tmp |= src << 1U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 15U; - src = *(in + 16 * 26 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 10U; - src = *(in + 16 * 27 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 35U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 21U; - src = *(in + 16 * 30 + i); - tmp |= src << 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 7U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 7U; - src = *(in + 16 * 34 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 18U; - src = *(in + 16 * 35 + i); - tmp |= src << 21U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 29U; - src = *(in + 16 * 38 + i); - tmp |= src << 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 15U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 1U; - src = *(in + 16 * 42 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 26U; - src = *(in + 16 * 43 + i); - tmp |= src << 13U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 37U; - src = *(in + 16 * 46 + i); - tmp |= src << 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 23U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 9U; - src = *(in + 16 * 50 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 34U; - src = *(in + 16 * 51 + i); - tmp |= src << 5U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 19U; - src = *(in + 16 * 54 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 31U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 17U; - src = *(in + 16 * 58 + i); - tmp |= src << 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 3U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 11U; - src = *(in + 16 * 62 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 14U; - src = *(in + 16 * 63 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out -= 608; - } -} -void pack_40bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 24U; - src = *(in + 16 * 2 + i); - tmp |= src << 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 8U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 8U; - src = *(in + 16 * 6 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 16U; - src = *(in + 16 * 7 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 24U; - src = *(in + 16 * 10 + i); - tmp |= src << 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 8U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 8U; - src = *(in + 16 * 14 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 16U; - src = *(in + 16 * 15 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 24U; - src = *(in + 16 * 18 + i); - tmp |= src << 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 8U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 8U; - src = *(in + 16 * 22 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 16U; - src = *(in + 16 * 23 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 24U; - src = *(in + 16 * 26 + i); - tmp |= src << 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 8U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 8U; - src = *(in + 16 * 30 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 16U; - src = *(in + 16 * 31 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 24U; - src = *(in + 16 * 34 + i); - tmp |= src << 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 8U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 8U; - src = *(in + 16 * 38 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 16U; - src = *(in + 16 * 39 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 24U; - src = *(in + 16 * 42 + i); - tmp |= src << 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 8U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 8U; - src = *(in + 16 * 46 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 16U; - src = *(in + 16 * 47 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 24U; - src = *(in + 16 * 50 + i); - tmp |= src << 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 8U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 8U; - src = *(in + 16 * 54 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 16U; - src = *(in + 16 * 55 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 24U; - src = *(in + 16 * 58 + i); - tmp |= src << 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 8U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 8U; - src = *(in + 16 * 62 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 16U; - src = *(in + 16 * 63 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out -= 624; - } -} -void pack_41bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 23U; - src = *(in + 16 * 2 + i); - tmp |= src << 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 5U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 13U; - src = *(in + 16 * 6 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 10U; - src = *(in + 16 * 7 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 33U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 15U; - src = *(in + 16 * 10 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 38U; - src = *(in + 16 * 11 + i); - tmp |= src << 3U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 21U; - src = *(in + 16 * 14 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 25U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 7U; - src = *(in + 16 * 18 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 30U; - src = *(in + 16 * 19 + i); - tmp |= src << 11U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 35U; - src = *(in + 16 * 22 + i); - tmp |= src << 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 17U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 1U; - src = *(in + 16 * 26 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 22U; - src = *(in + 16 * 27 + i); - tmp |= src << 19U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 27U; - src = *(in + 16 * 30 + i); - tmp |= src << 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 9U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 9U; - src = *(in + 16 * 34 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 14U; - src = *(in + 16 * 35 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 37U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 19U; - src = *(in + 16 * 38 + i); - tmp |= src << 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 1U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 17U; - src = *(in + 16 * 42 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 29U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 11U; - src = *(in + 16 * 46 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 34U; - src = *(in + 16 * 47 + i); - tmp |= src << 7U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 39U; - src = *(in + 16 * 50 + i); - tmp |= src << 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 21U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 3U; - src = *(in + 16 * 54 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 26U; - src = *(in + 16 * 55 + i); - tmp |= src << 15U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 31U; - src = *(in + 16 * 58 + i); - tmp |= src << 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 13U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 5U; - src = *(in + 16 * 62 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 18U; - src = *(in + 16 * 63 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_42bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 22U; - src = *(in + 16 * 2 + i); - tmp |= src << 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 2U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 18U; - src = *(in + 16 * 6 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 26U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 6U; - src = *(in + 16 * 10 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 28U; - src = *(in + 16 * 11 + i); - tmp |= src << 14U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 30U; - src = *(in + 16 * 14 + i); - tmp |= src << 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 10U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 10U; - src = *(in + 16 * 18 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 12U; - src = *(in + 16 * 19 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 34U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 14U; - src = *(in + 16 * 22 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 36U; - src = *(in + 16 * 23 + i); - tmp |= src << 6U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 38U; - src = *(in + 16 * 26 + i); - tmp |= src << 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 18U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 2U; - src = *(in + 16 * 30 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 20U; - src = *(in + 16 * 31 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 22U; - src = *(in + 16 * 34 + i); - tmp |= src << 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 2U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 18U; - src = *(in + 16 * 38 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 26U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 6U; - src = *(in + 16 * 42 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 28U; - src = *(in + 16 * 43 + i); - tmp |= src << 14U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 30U; - src = *(in + 16 * 46 + i); - tmp |= src << 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 10U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 10U; - src = *(in + 16 * 50 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 12U; - src = *(in + 16 * 51 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 34U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 14U; - src = *(in + 16 * 54 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 36U; - src = *(in + 16 * 55 + i); - tmp |= src << 6U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 38U; - src = *(in + 16 * 58 + i); - tmp |= src << 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 18U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 2U; - src = *(in + 16 * 62 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 20U; - src = *(in + 16 * 63 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out -= 656; - } -} -void pack_43bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 21U; - src = *(in + 16 * 2 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 42U; - src = *(in + 16 * 3 + i); - tmp |= src << 1U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 41U; - src = *(in + 16 * 6 + i); - tmp |= src << 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 19U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 3U; - src = *(in + 16 * 10 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 18U; - src = *(in + 16 * 11 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 39U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 17U; - src = *(in + 16 * 14 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 38U; - src = *(in + 16 * 15 + i); - tmp |= src << 5U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 37U; - src = *(in + 16 * 18 + i); - tmp |= src << 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 15U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 7U; - src = *(in + 16 * 22 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 14U; - src = *(in + 16 * 23 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 35U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 13U; - src = *(in + 16 * 26 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 34U; - src = *(in + 16 * 27 + i); - tmp |= src << 9U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 33U; - src = *(in + 16 * 30 + i); - tmp |= src << 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 11U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 11U; - src = *(in + 16 * 34 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 31U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 9U; - src = *(in + 16 * 38 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 30U; - src = *(in + 16 * 39 + i); - tmp |= src << 13U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 29U; - src = *(in + 16 * 42 + i); - tmp |= src << 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 7U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 15U; - src = *(in + 16 * 46 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 27U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 5U; - src = *(in + 16 * 50 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 26U; - src = *(in + 16 * 51 + i); - tmp |= src << 17U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 25U; - src = *(in + 16 * 54 + i); - tmp |= src << 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 3U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 19U; - src = *(in + 16 * 58 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 23U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 1U; - src = *(in + 16 * 62 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 22U; - src = *(in + 16 * 63 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out -= 672; - } -} -void pack_44bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 20U; - src = *(in + 16 * 2 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 40U; - src = *(in + 16 * 3 + i); - tmp |= src << 4U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 36U; - src = *(in + 16 * 6 + i); - tmp |= src << 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 12U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 12U; - src = *(in + 16 * 10 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 28U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 4U; - src = *(in + 16 * 14 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 24U; - src = *(in + 16 * 15 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 20U; - src = *(in + 16 * 18 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 40U; - src = *(in + 16 * 19 + i); - tmp |= src << 4U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 36U; - src = *(in + 16 * 22 + i); - tmp |= src << 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 12U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 12U; - src = *(in + 16 * 26 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 28U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 4U; - src = *(in + 16 * 30 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 24U; - src = *(in + 16 * 31 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 20U; - src = *(in + 16 * 34 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 40U; - src = *(in + 16 * 35 + i); - tmp |= src << 4U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 36U; - src = *(in + 16 * 38 + i); - tmp |= src << 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 12U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 12U; - src = *(in + 16 * 42 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 28U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 4U; - src = *(in + 16 * 46 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 24U; - src = *(in + 16 * 47 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 20U; - src = *(in + 16 * 50 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 40U; - src = *(in + 16 * 51 + i); - tmp |= src << 4U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 36U; - src = *(in + 16 * 54 + i); - tmp |= src << 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 12U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 12U; - src = *(in + 16 * 58 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 28U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 4U; - src = *(in + 16 * 62 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 24U; - src = *(in + 16 * 63 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out -= 688; - } -} -void pack_45bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 19U; - src = *(in + 16 * 2 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 38U; - src = *(in + 16 * 3 + i); - tmp |= src << 7U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 31U; - src = *(in + 16 * 6 + i); - tmp |= src << 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 5U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 43U; - src = *(in + 16 * 10 + i); - tmp |= src << 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 17U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 9U; - src = *(in + 16 * 14 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 29U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 3U; - src = *(in + 16 * 18 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 22U; - src = *(in + 16 * 19 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 41U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 15U; - src = *(in + 16 * 22 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 34U; - src = *(in + 16 * 23 + i); - tmp |= src << 11U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 27U; - src = *(in + 16 * 26 + i); - tmp |= src << 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 1U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 39U; - src = *(in + 16 * 30 + i); - tmp |= src << 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 13U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 13U; - src = *(in + 16 * 34 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 25U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 1U; - src = *(in + 16 * 38 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 18U; - src = *(in + 16 * 39 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 37U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 11U; - src = *(in + 16 * 42 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 30U; - src = *(in + 16 * 43 + i); - tmp |= src << 15U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 23U; - src = *(in + 16 * 46 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 42U; - src = *(in + 16 * 47 + i); - tmp |= src << 3U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 35U; - src = *(in + 16 * 50 + i); - tmp |= src << 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 9U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 17U; - src = *(in + 16 * 54 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 21U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 5U; - src = *(in + 16 * 58 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 14U; - src = *(in + 16 * 59 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 33U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 7U; - src = *(in + 16 * 62 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 26U; - src = *(in + 16 * 63 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out -= 704; - } -} -void pack_46bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 18U; - src = *(in + 16 * 2 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 36U; - src = *(in + 16 * 3 + i); - tmp |= src << 10U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 26U; - src = *(in + 16 * 6 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 44U; - src = *(in + 16 * 7 + i); - tmp |= src << 2U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 34U; - src = *(in + 16 * 10 + i); - tmp |= src << 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 6U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 42U; - src = *(in + 16 * 14 + i); - tmp |= src << 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 14U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 14U; - src = *(in + 16 * 18 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 22U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 6U; - src = *(in + 16 * 22 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 30U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 2U; - src = *(in + 16 * 26 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 20U; - src = *(in + 16 * 27 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 38U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 10U; - src = *(in + 16 * 30 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 28U; - src = *(in + 16 * 31 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 18U; - src = *(in + 16 * 34 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 36U; - src = *(in + 16 * 35 + i); - tmp |= src << 10U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 26U; - src = *(in + 16 * 38 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 44U; - src = *(in + 16 * 39 + i); - tmp |= src << 2U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 34U; - src = *(in + 16 * 42 + i); - tmp |= src << 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 6U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 42U; - src = *(in + 16 * 46 + i); - tmp |= src << 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 14U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 14U; - src = *(in + 16 * 50 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 22U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 6U; - src = *(in + 16 * 54 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 30U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 2U; - src = *(in + 16 * 58 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 20U; - src = *(in + 16 * 59 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 38U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 10U; - src = *(in + 16 * 62 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 28U; - src = *(in + 16 * 63 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out -= 720; - } -} -void pack_47bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 17U; - src = *(in + 16 * 2 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 34U; - src = *(in + 16 * 3 + i); - tmp |= src << 13U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 21U; - src = *(in + 16 * 6 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 38U; - src = *(in + 16 * 7 + i); - tmp |= src << 9U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 25U; - src = *(in + 16 * 10 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 42U; - src = *(in + 16 * 11 + i); - tmp |= src << 5U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 29U; - src = *(in + 16 * 14 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 46U; - src = *(in + 16 * 15 + i); - tmp |= src << 1U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 33U; - src = *(in + 16 * 18 + i); - tmp |= src << 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 3U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 37U; - src = *(in + 16 * 22 + i); - tmp |= src << 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 7U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 41U; - src = *(in + 16 * 26 + i); - tmp |= src << 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 11U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 45U; - src = *(in + 16 * 30 + i); - tmp |= src << 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 15U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 15U; - src = *(in + 16 * 34 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 19U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 11U; - src = *(in + 16 * 38 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 23U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 7U; - src = *(in + 16 * 42 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 27U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 3U; - src = *(in + 16 * 46 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 31U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 1U; - src = *(in + 16 * 50 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 18U; - src = *(in + 16 * 51 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 35U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 5U; - src = *(in + 16 * 54 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 22U; - src = *(in + 16 * 55 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 39U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 9U; - src = *(in + 16 * 58 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 26U; - src = *(in + 16 * 59 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 43U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 13U; - src = *(in + 16 * 62 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 30U; - src = *(in + 16 * 63 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out -= 736; - } -} -void pack_48bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 16U; - src = *(in + 16 * 2 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 32U; - src = *(in + 16 * 3 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - src = *(in + 16 * 5 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 16U; - src = *(in + 16 * 6 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 32U; - src = *(in + 16 * 7 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 16U; - src = *(in + 16 * 10 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 32U; - src = *(in + 16 * 11 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - src = *(in + 16 * 13 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 16U; - src = *(in + 16 * 14 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 32U; - src = *(in + 16 * 15 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 16U; - src = *(in + 16 * 18 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 32U; - src = *(in + 16 * 19 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - src = *(in + 16 * 21 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 16U; - src = *(in + 16 * 22 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 32U; - src = *(in + 16 * 23 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 16U; - src = *(in + 16 * 26 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 32U; - src = *(in + 16 * 27 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - src = *(in + 16 * 29 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 16U; - src = *(in + 16 * 30 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 32U; - src = *(in + 16 * 31 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 16U; - src = *(in + 16 * 34 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 32U; - src = *(in + 16 * 35 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - src = *(in + 16 * 37 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 16U; - src = *(in + 16 * 38 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 32U; - src = *(in + 16 * 39 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 16U; - src = *(in + 16 * 42 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 32U; - src = *(in + 16 * 43 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - src = *(in + 16 * 45 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 16U; - src = *(in + 16 * 46 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 32U; - src = *(in + 16 * 47 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 16U; - src = *(in + 16 * 50 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 32U; - src = *(in + 16 * 51 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - src = *(in + 16 * 53 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 16U; - src = *(in + 16 * 54 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 32U; - src = *(in + 16 * 55 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 16U; - src = *(in + 16 * 58 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 32U; - src = *(in + 16 * 59 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - src = *(in + 16 * 61 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 16U; - src = *(in + 16 * 62 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 32U; - src = *(in + 16 * 63 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out -= 752; - } -} -void pack_49bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 15U; - src = *(in + 16 * 2 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 30U; - src = *(in + 16 * 3 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 45U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 11U; - src = *(in + 16 * 6 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 26U; - src = *(in + 16 * 7 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 41U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 7U; - src = *(in + 16 * 10 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 22U; - src = *(in + 16 * 11 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 37U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 3U; - src = *(in + 16 * 14 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 18U; - src = *(in + 16 * 15 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 33U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 1U; - src = *(in + 16 * 18 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 29U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 44U; - src = *(in + 16 * 21 + i); - tmp |= src << 5U; - src = *(in + 16 * 22 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 25U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 9U; - src = *(in + 16 * 26 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 21U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 36U; - src = *(in + 16 * 29 + i); - tmp |= src << 13U; - src = *(in + 16 * 30 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 17U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 47U; - src = *(in + 16 * 34 + i); - tmp |= src << 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 13U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 43U; - src = *(in + 16 * 38 + i); - tmp |= src << 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 9U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 39U; - src = *(in + 16 * 42 + i); - tmp |= src << 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 5U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 35U; - src = *(in + 16 * 46 + i); - tmp |= src << 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 1U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 31U; - src = *(in + 16 * 50 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 46U; - src = *(in + 16 * 51 + i); - tmp |= src << 3U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 27U; - src = *(in + 16 * 54 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 42U; - src = *(in + 16 * 55 + i); - tmp |= src << 7U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 23U; - src = *(in + 16 * 58 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 38U; - src = *(in + 16 * 59 + i); - tmp |= src << 11U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 19U; - src = *(in + 16 * 62 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 34U; - src = *(in + 16 * 63 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_50bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 14U; - src = *(in + 16 * 2 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 28U; - src = *(in + 16 * 3 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 42U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 6U; - src = *(in + 16 * 6 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 20U; - src = *(in + 16 * 7 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 34U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 2U; - src = *(in + 16 * 10 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 26U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 40U; - src = *(in + 16 * 13 + i); - tmp |= src << 10U; - src = *(in + 16 * 14 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 18U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 46U; - src = *(in + 16 * 18 + i); - tmp |= src << 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 10U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 38U; - src = *(in + 16 * 22 + i); - tmp |= src << 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 2U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 30U; - src = *(in + 16 * 26 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 44U; - src = *(in + 16 * 27 + i); - tmp |= src << 6U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 22U; - src = *(in + 16 * 30 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 36U; - src = *(in + 16 * 31 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 14U; - src = *(in + 16 * 34 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 28U; - src = *(in + 16 * 35 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 42U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 6U; - src = *(in + 16 * 38 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 20U; - src = *(in + 16 * 39 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 34U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 2U; - src = *(in + 16 * 42 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 26U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 40U; - src = *(in + 16 * 45 + i); - tmp |= src << 10U; - src = *(in + 16 * 46 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 18U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 46U; - src = *(in + 16 * 50 + i); - tmp |= src << 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 10U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 38U; - src = *(in + 16 * 54 + i); - tmp |= src << 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 2U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 30U; - src = *(in + 16 * 58 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 44U; - src = *(in + 16 * 59 + i); - tmp |= src << 6U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 22U; - src = *(in + 16 * 62 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 36U; - src = *(in + 16 * 63 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out -= 784; - } -} -void pack_51bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 13U; - src = *(in + 16 * 2 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 26U; - src = *(in + 16 * 3 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 39U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 1U; - src = *(in + 16 * 6 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 27U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 11U; - src = *(in + 16 * 10 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 15U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 41U; - src = *(in + 16 * 14 + i); - tmp |= src << 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 3U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 29U; - src = *(in + 16 * 18 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 42U; - src = *(in + 16 * 19 + i); - tmp |= src << 9U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 17U; - src = *(in + 16 * 22 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 30U; - src = *(in + 16 * 23 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 43U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 5U; - src = *(in + 16 * 26 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 31U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 44U; - src = *(in + 16 * 29 + i); - tmp |= src << 7U; - src = *(in + 16 * 30 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 19U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 45U; - src = *(in + 16 * 34 + i); - tmp |= src << 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 7U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 33U; - src = *(in + 16 * 38 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 46U; - src = *(in + 16 * 39 + i); - tmp |= src << 5U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 21U; - src = *(in + 16 * 42 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 34U; - src = *(in + 16 * 43 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 47U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 9U; - src = *(in + 16 * 46 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 22U; - src = *(in + 16 * 47 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 35U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 3U; - src = *(in + 16 * 50 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 23U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 36U; - src = *(in + 16 * 53 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 49U; - src = *(in + 16 * 54 + i); - tmp |= src << 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 11U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 37U; - src = *(in + 16 * 58 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 50U; - src = *(in + 16 * 59 + i); - tmp |= src << 1U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 25U; - src = *(in + 16 * 62 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 38U; - src = *(in + 16 * 63 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out -= 800; - } -} -void pack_52bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 12U; - src = *(in + 16 * 2 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 24U; - src = *(in + 16 * 3 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 36U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 48U; - src = *(in + 16 * 5 + i); - tmp |= src << 4U; - src = *(in + 16 * 6 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 20U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 44U; - src = *(in + 16 * 10 + i); - tmp |= src << 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 4U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 28U; - src = *(in + 16 * 14 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 40U; - src = *(in + 16 * 15 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 12U; - src = *(in + 16 * 18 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 24U; - src = *(in + 16 * 19 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 36U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 48U; - src = *(in + 16 * 21 + i); - tmp |= src << 4U; - src = *(in + 16 * 22 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 20U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 44U; - src = *(in + 16 * 26 + i); - tmp |= src << 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 4U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 28U; - src = *(in + 16 * 30 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 40U; - src = *(in + 16 * 31 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 12U; - src = *(in + 16 * 34 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 24U; - src = *(in + 16 * 35 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 36U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 48U; - src = *(in + 16 * 37 + i); - tmp |= src << 4U; - src = *(in + 16 * 38 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 20U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 44U; - src = *(in + 16 * 42 + i); - tmp |= src << 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 4U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 28U; - src = *(in + 16 * 46 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 40U; - src = *(in + 16 * 47 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 12U; - src = *(in + 16 * 50 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 24U; - src = *(in + 16 * 51 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 36U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 48U; - src = *(in + 16 * 53 + i); - tmp |= src << 4U; - src = *(in + 16 * 54 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 20U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 44U; - src = *(in + 16 * 58 + i); - tmp |= src << 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 4U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 28U; - src = *(in + 16 * 62 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 40U; - src = *(in + 16 * 63 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out -= 816; - } -} -void pack_53bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 11U; - src = *(in + 16 * 2 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 22U; - src = *(in + 16 * 3 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 33U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 44U; - src = *(in + 16 * 5 + i); - tmp |= src << 9U; - src = *(in + 16 * 6 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 13U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 35U; - src = *(in + 16 * 10 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 46U; - src = *(in + 16 * 11 + i); - tmp |= src << 7U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 15U; - src = *(in + 16 * 14 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 26U; - src = *(in + 16 * 15 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 37U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 5U; - src = *(in + 16 * 18 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 17U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 39U; - src = *(in + 16 * 22 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 50U; - src = *(in + 16 * 23 + i); - tmp |= src << 3U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 19U; - src = *(in + 16 * 26 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 30U; - src = *(in + 16 * 27 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 41U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 52U; - src = *(in + 16 * 29 + i); - tmp |= src << 1U; - src = *(in + 16 * 30 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 21U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 43U; - src = *(in + 16 * 34 + i); - tmp |= src << 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 1U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 23U; - src = *(in + 16 * 38 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 34U; - src = *(in + 16 * 39 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 45U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 3U; - src = *(in + 16 * 42 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 25U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 47U; - src = *(in + 16 * 46 + i); - tmp |= src << 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 5U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 27U; - src = *(in + 16 * 50 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 38U; - src = *(in + 16 * 51 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 49U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 7U; - src = *(in + 16 * 54 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 29U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 51U; - src = *(in + 16 * 58 + i); - tmp |= src << 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 9U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 31U; - src = *(in + 16 * 62 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 42U; - src = *(in + 16 * 63 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out -= 832; - } -} -void pack_54bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 10U; - src = *(in + 16 * 2 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 30U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 40U; - src = *(in + 16 * 5 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 50U; - src = *(in + 16 * 6 + i); - tmp |= src << 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 6U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 26U; - src = *(in + 16 * 10 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 36U; - src = *(in + 16 * 11 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 46U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 2U; - src = *(in + 16 * 14 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 22U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 42U; - src = *(in + 16 * 18 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 52U; - src = *(in + 16 * 19 + i); - tmp |= src << 2U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 18U; - src = *(in + 16 * 22 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 28U; - src = *(in + 16 * 23 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 38U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 6U; - src = *(in + 16 * 26 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 14U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 34U; - src = *(in + 16 * 30 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 44U; - src = *(in + 16 * 31 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 10U; - src = *(in + 16 * 34 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 30U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 40U; - src = *(in + 16 * 37 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 50U; - src = *(in + 16 * 38 + i); - tmp |= src << 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 6U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 26U; - src = *(in + 16 * 42 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 36U; - src = *(in + 16 * 43 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 46U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 2U; - src = *(in + 16 * 46 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 22U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 42U; - src = *(in + 16 * 50 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 52U; - src = *(in + 16 * 51 + i); - tmp |= src << 2U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 18U; - src = *(in + 16 * 54 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 28U; - src = *(in + 16 * 55 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 38U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 6U; - src = *(in + 16 * 58 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 14U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 34U; - src = *(in + 16 * 62 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 44U; - src = *(in + 16 * 63 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out -= 848; - } -} -void pack_55bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 9U; - src = *(in + 16 * 2 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 27U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 45U; - src = *(in + 16 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 54U; - src = *(in + 16 * 7 + i); - tmp |= src << 1U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 17U; - src = *(in + 16 * 10 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 26U; - src = *(in + 16 * 11 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 35U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 44U; - src = *(in + 16 * 13 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 53U; - src = *(in + 16 * 14 + i); - tmp |= src << 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 7U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 25U; - src = *(in + 16 * 18 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 34U; - src = *(in + 16 * 19 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 43U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 52U; - src = *(in + 16 * 21 + i); - tmp |= src << 3U; - src = *(in + 16 * 22 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 15U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 33U; - src = *(in + 16 * 26 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 42U; - src = *(in + 16 * 27 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 51U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 5U; - src = *(in + 16 * 30 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 23U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 41U; - src = *(in + 16 * 34 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 50U; - src = *(in + 16 * 35 + i); - tmp |= src << 5U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 13U; - src = *(in + 16 * 38 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 31U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 49U; - src = *(in + 16 * 42 + i); - tmp |= src << 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 3U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 21U; - src = *(in + 16 * 46 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 30U; - src = *(in + 16 * 47 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 39U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 7U; - src = *(in + 16 * 50 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 11U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 29U; - src = *(in + 16 * 54 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 38U; - src = *(in + 16 * 55 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 47U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 1U; - src = *(in + 16 * 58 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 19U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 37U; - src = *(in + 16 * 62 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 46U; - src = *(in + 16 * 63 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out -= 864; - } -} -void pack_56bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 8U; - src = *(in + 16 * 2 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 24U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 40U; - src = *(in + 16 * 6 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 48U; - src = *(in + 16 * 7 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 8U; - src = *(in + 16 * 10 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 24U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 40U; - src = *(in + 16 * 14 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 48U; - src = *(in + 16 * 15 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 8U; - src = *(in + 16 * 18 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 24U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 40U; - src = *(in + 16 * 22 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 48U; - src = *(in + 16 * 23 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 8U; - src = *(in + 16 * 26 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 24U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 40U; - src = *(in + 16 * 30 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 48U; - src = *(in + 16 * 31 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 8U; - src = *(in + 16 * 34 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 24U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 40U; - src = *(in + 16 * 38 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 48U; - src = *(in + 16 * 39 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 8U; - src = *(in + 16 * 42 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 24U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 40U; - src = *(in + 16 * 46 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 48U; - src = *(in + 16 * 47 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 8U; - src = *(in + 16 * 50 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 24U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 40U; - src = *(in + 16 * 54 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 48U; - src = *(in + 16 * 55 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 8U; - src = *(in + 16 * 58 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 24U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 40U; - src = *(in + 16 * 62 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 48U; - src = *(in + 16 * 63 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out -= 880; - } -} -void pack_57bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 7U; - src = *(in + 16 * 2 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 21U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 35U; - src = *(in + 16 * 6 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 42U; - src = *(in + 16 * 7 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 49U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 56U; - src = *(in + 16 * 9 + i); - tmp |= src << 1U; - src = *(in + 16 * 10 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 13U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 27U; - src = *(in + 16 * 14 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 34U; - src = *(in + 16 * 15 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 41U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 55U; - src = *(in + 16 * 18 + i); - tmp |= src << 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 5U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 19U; - src = *(in + 16 * 22 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 26U; - src = *(in + 16 * 23 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 33U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 47U; - src = *(in + 16 * 26 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 54U; - src = *(in + 16 * 27 + i); - tmp |= src << 3U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 11U; - src = *(in + 16 * 30 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 25U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 39U; - src = *(in + 16 * 34 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 46U; - src = *(in + 16 * 35 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 53U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 3U; - src = *(in + 16 * 38 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 17U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 31U; - src = *(in + 16 * 42 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 38U; - src = *(in + 16 * 43 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 45U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 52U; - src = *(in + 16 * 45 + i); - tmp |= src << 5U; - src = *(in + 16 * 46 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 9U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 23U; - src = *(in + 16 * 50 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 30U; - src = *(in + 16 * 51 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 37U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 44U; - src = *(in + 16 * 53 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 51U; - src = *(in + 16 * 54 + i); - tmp |= src << 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 1U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 15U; - src = *(in + 16 * 58 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 29U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 43U; - src = *(in + 16 * 62 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 50U; - src = *(in + 16 * 63 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 896; - } -} -void pack_58bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 6U; - src = *(in + 16 * 2 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 18U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 30U; - src = *(in + 16 * 6 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 36U; - src = *(in + 16 * 7 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 42U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 54U; - src = *(in + 16 * 10 + i); - tmp |= src << 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 2U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 14U; - src = *(in + 16 * 14 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 26U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 38U; - src = *(in + 16 * 18 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 44U; - src = *(in + 16 * 19 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 50U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 56U; - src = *(in + 16 * 21 + i); - tmp |= src << 2U; - src = *(in + 16 * 22 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 10U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 22U; - src = *(in + 16 * 26 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 28U; - src = *(in + 16 * 27 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 34U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 46U; - src = *(in + 16 * 30 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 52U; - src = *(in + 16 * 31 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 6U; - src = *(in + 16 * 34 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 18U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 30U; - src = *(in + 16 * 38 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 36U; - src = *(in + 16 * 39 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 42U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 54U; - src = *(in + 16 * 42 + i); - tmp |= src << 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 2U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 14U; - src = *(in + 16 * 46 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 26U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 38U; - src = *(in + 16 * 50 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 44U; - src = *(in + 16 * 51 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 50U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 56U; - src = *(in + 16 * 53 + i); - tmp |= src << 2U; - src = *(in + 16 * 54 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 10U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 22U; - src = *(in + 16 * 58 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 28U; - src = *(in + 16 * 59 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 34U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 46U; - src = *(in + 16 * 62 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 52U; - src = *(in + 16 * 63 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 912; - } -} -void pack_59bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 5U; - src = *(in + 16 * 2 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 15U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 25U; - src = *(in + 16 * 6 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 30U; - src = *(in + 16 * 7 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 35U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 45U; - src = *(in + 16 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 50U; - src = *(in + 16 * 11 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 55U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 1U; - src = *(in + 16 * 14 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 11U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 21U; - src = *(in + 16 * 18 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 31U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 41U; - src = *(in + 16 * 22 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 46U; - src = *(in + 16 * 23 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 51U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 56U; - src = *(in + 16 * 25 + i); - tmp |= src << 3U; - src = *(in + 16 * 26 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 7U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 17U; - src = *(in + 16 * 30 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 27U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 37U; - src = *(in + 16 * 34 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 42U; - src = *(in + 16 * 35 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 47U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 52U; - src = *(in + 16 * 37 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 57U; - src = *(in + 16 * 38 + i); - tmp |= src << 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 3U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 13U; - src = *(in + 16 * 42 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 23U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 33U; - src = *(in + 16 * 46 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 38U; - src = *(in + 16 * 47 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 43U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 53U; - src = *(in + 16 * 50 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 58U; - src = *(in + 16 * 51 + i); - tmp |= src << 1U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 9U; - src = *(in + 16 * 54 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 19U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 29U; - src = *(in + 16 * 58 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 34U; - src = *(in + 16 * 59 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 39U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 44U; - src = *(in + 16 * 61 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 49U; - src = *(in + 16 * 62 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 54U; - src = *(in + 16 * 63 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 928; - } -} -void pack_60bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 4U; - src = *(in + 16 * 2 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 12U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 20U; - src = *(in + 16 * 6 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 28U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 36U; - src = *(in + 16 * 10 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 40U; - src = *(in + 16 * 11 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 44U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 48U; - src = *(in + 16 * 13 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 52U; - src = *(in + 16 * 14 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 56U; - src = *(in + 16 * 15 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 4U; - src = *(in + 16 * 18 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 12U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 20U; - src = *(in + 16 * 22 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 28U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 36U; - src = *(in + 16 * 26 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 40U; - src = *(in + 16 * 27 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 44U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 48U; - src = *(in + 16 * 29 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 52U; - src = *(in + 16 * 30 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 56U; - src = *(in + 16 * 31 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 4U; - src = *(in + 16 * 34 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 12U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 20U; - src = *(in + 16 * 38 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 28U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 36U; - src = *(in + 16 * 42 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 40U; - src = *(in + 16 * 43 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 44U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 48U; - src = *(in + 16 * 45 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 52U; - src = *(in + 16 * 46 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 56U; - src = *(in + 16 * 47 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 4U; - src = *(in + 16 * 50 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 12U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 20U; - src = *(in + 16 * 54 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 28U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 36U; - src = *(in + 16 * 58 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 40U; - src = *(in + 16 * 59 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 44U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 48U; - src = *(in + 16 * 61 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 52U; - src = *(in + 16 * 62 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 56U; - src = *(in + 16 * 63 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 944; - } -} -void pack_61bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 3U; - src = *(in + 16 * 2 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 9U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 15U; - src = *(in + 16 * 6 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 21U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 27U; - src = *(in + 16 * 10 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 30U; - src = *(in + 16 * 11 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 33U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 39U; - src = *(in + 16 * 14 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 42U; - src = *(in + 16 * 15 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 45U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 51U; - src = *(in + 16 * 18 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 54U; - src = *(in + 16 * 19 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 57U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 60U; - src = *(in + 16 * 21 + i); - tmp |= src << 1U; - src = *(in + 16 * 22 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 5U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 11U; - src = *(in + 16 * 26 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 17U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 23U; - src = *(in + 16 * 30 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 29U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 35U; - src = *(in + 16 * 34 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 38U; - src = *(in + 16 * 35 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 41U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 47U; - src = *(in + 16 * 38 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 50U; - src = *(in + 16 * 39 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 53U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 56U; - src = *(in + 16 * 41 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 59U; - src = *(in + 16 * 42 + i); - tmp |= src << 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 1U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 7U; - src = *(in + 16 * 46 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 13U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 19U; - src = *(in + 16 * 50 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 25U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 31U; - src = *(in + 16 * 54 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 34U; - src = *(in + 16 * 55 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 37U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 43U; - src = *(in + 16 * 58 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 46U; - src = *(in + 16 * 59 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 49U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 52U; - src = *(in + 16 * 61 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 55U; - src = *(in + 16 * 62 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 58U; - src = *(in + 16 * 63 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 960; - } -} -void pack_62bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 2U; - src = *(in + 16 * 2 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 6U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 10U; - src = *(in + 16 * 6 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 14U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 18U; - src = *(in + 16 * 10 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 22U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 26U; - src = *(in + 16 * 14 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 30U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 34U; - src = *(in + 16 * 18 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 36U; - src = *(in + 16 * 19 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 38U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 42U; - src = *(in + 16 * 22 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 44U; - src = *(in + 16 * 23 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 46U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 50U; - src = *(in + 16 * 26 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 52U; - src = *(in + 16 * 27 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 54U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 56U; - src = *(in + 16 * 29 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 58U; - src = *(in + 16 * 30 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 60U; - src = *(in + 16 * 31 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 2U; - src = *(in + 16 * 34 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 6U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 10U; - src = *(in + 16 * 38 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 14U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 18U; - src = *(in + 16 * 42 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 22U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 26U; - src = *(in + 16 * 46 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 30U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 34U; - src = *(in + 16 * 50 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 36U; - src = *(in + 16 * 51 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 38U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 42U; - src = *(in + 16 * 54 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 44U; - src = *(in + 16 * 55 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 46U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 50U; - src = *(in + 16 * 58 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 52U; - src = *(in + 16 * 59 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 54U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 56U; - src = *(in + 16 * 61 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 58U; - src = *(in + 16 * 62 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 60U; - src = *(in + 16 * 63 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 976; - } -} -void pack_63bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 1U; - src = *(in + 16 * 2 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 3U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 5U; - src = *(in + 16 * 6 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 7U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 9U; - src = *(in + 16 * 10 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 11U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 13U; - src = *(in + 16 * 14 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 15U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 17U; - src = *(in + 16 * 18 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 19U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 21U; - src = *(in + 16 * 22 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 23U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 25U; - src = *(in + 16 * 26 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 27U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 29U; - src = *(in + 16 * 30 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 31U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 33U; - src = *(in + 16 * 34 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 34U; - src = *(in + 16 * 35 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 35U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 37U; - src = *(in + 16 * 38 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 38U; - src = *(in + 16 * 39 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 39U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 41U; - src = *(in + 16 * 42 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 42U; - src = *(in + 16 * 43 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 43U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 45U; - src = *(in + 16 * 46 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 46U; - src = *(in + 16 * 47 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 47U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 49U; - src = *(in + 16 * 50 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 50U; - src = *(in + 16 * 51 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 51U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 52U; - src = *(in + 16 * 53 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 53U; - src = *(in + 16 * 54 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 54U; - src = *(in + 16 * 55 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 55U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 56U; - src = *(in + 16 * 57 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 57U; - src = *(in + 16 * 58 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 58U; - src = *(in + 16 * 59 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 59U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 60U; - src = *(in + 16 * 61 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 61U; - src = *(in + 16 * 62 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 62U; - src = *(in + 16 * 63 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 992; - } -} -void pack_64bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 63 + i); - tmp = src; - *(out + i) = tmp; - out -= 1008; - } -} - -void Compress08(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - uint8_t* __restrict in = 0; - uint8_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - pack_1bit_8simdWidth(in, out); - break; - case 2: - pack_2bit_8simdWidth(in, out); - break; - case 3: - pack_3bit_8simdWidth(in, out); - break; - case 4: - pack_4bit_8simdWidth(in, out); - break; - case 5: - pack_5bit_8simdWidth(in, out); - break; - case 6: - pack_6bit_8simdWidth(in, out); - break; - case 7: - pack_7bit_8simdWidth(in, out); - break; - case 8: - pack_8bit_8simdWidth(in, out); - break; - } -} - -void Compress16(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - uint16_t* __restrict in = 0; - uint16_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - pack_1bit_16simdWidth(in, out); - break; - case 2: - pack_2bit_16simdWidth(in, out); - break; - case 3: - pack_3bit_16simdWidth(in, out); - break; - case 4: - pack_4bit_16simdWidth(in, out); - break; - case 5: - pack_5bit_16simdWidth(in, out); - break; - case 6: - pack_6bit_16simdWidth(in, out); - break; - case 7: - pack_7bit_16simdWidth(in, out); - break; - case 8: - pack_8bit_16simdWidth(in, out); - break; - case 9: - pack_9bit_16simdWidth(in, out); - break; - case 10: - pack_10bit_16simdWidth(in, out); - break; - case 11: - pack_11bit_16simdWidth(in, out); - break; - case 12: - pack_12bit_16simdWidth(in, out); - break; - case 13: - pack_13bit_16simdWidth(in, out); - break; - case 14: - pack_14bit_16simdWidth(in, out); - break; - case 15: - pack_15bit_16simdWidth(in, out); - break; - case 16: - pack_16bit_16simdWidth(in, out); - break; - } -} -void Compress32(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - uint32_t* __restrict in = 0; - uint32_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - pack_1bit_32simdWidth(in, out); - break; - case 2: - pack_2bit_32simdWidth(in, out); - break; - case 3: - pack_3bit_32simdWidth(in, out); - break; - case 4: - pack_4bit_32simdWidth(in, out); - break; - case 5: - pack_5bit_32simdWidth(in, out); - break; - case 6: - pack_6bit_32simdWidth(in, out); - break; - case 7: - pack_7bit_32simdWidth(in, out); - break; - case 8: - pack_8bit_32simdWidth(in, out); - break; - case 9: - pack_9bit_32simdWidth(in, out); - break; - case 10: - pack_10bit_32simdWidth(in, out); - break; - case 11: - pack_11bit_32simdWidth(in, out); - break; - case 12: - pack_12bit_32simdWidth(in, out); - break; - case 13: - pack_13bit_32simdWidth(in, out); - break; - case 14: - pack_14bit_32simdWidth(in, out); - break; - case 15: - pack_15bit_32simdWidth(in, out); - break; - case 16: - pack_16bit_32simdWidth(in, out); - break; - case 17: - pack_17bit_32simdWidth(in, out); - break; - case 18: - pack_18bit_32simdWidth(in, out); - break; - case 19: - pack_19bit_32simdWidth(in, out); - break; - case 20: - pack_20bit_32simdWidth(in, out); - break; - case 21: - pack_21bit_32simdWidth(in, out); - break; - case 22: - pack_22bit_32simdWidth(in, out); - break; - case 23: - pack_23bit_32simdWidth(in, out); - break; - case 24: - pack_24bit_32simdWidth(in, out); - break; - case 25: - pack_25bit_32simdWidth(in, out); - break; - case 26: - pack_26bit_32simdWidth(in, out); - break; - case 27: - pack_27bit_32simdWidth(in, out); - break; - case 28: - pack_28bit_32simdWidth(in, out); - break; - case 29: - pack_29bit_32simdWidth(in, out); - break; - case 30: - pack_30bit_32simdWidth(in, out); - break; - case 31: - pack_31bit_32simdWidth(in, out); - break; - case 32: - pack_32bit_32simdWidth(in, out); - break; - } -} - -void Compress64(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - // auto *bit_col = reinterpret_cast(state.col1); - // auto out = reinterpret_cast(state.data); - // auto in = reinterpret_cast(state.data); - - uint8_t bit_c = 0; - uint64_t* in = 0; - uint64_t* out = 0; - - switch (bit_c) { - case 0: - break; - case 1: - pack_1bit_16simdWidth(in, out); - break; - case 2: - pack_2bit_64simdWidth(in, out); - break; - case 3: - pack_3bit_64simdWidth(in, out); - break; - case 4: - pack_4bit_64simdWidth(in, out); - break; - case 5: - pack_5bit_64simdWidth(in, out); - break; - case 6: - pack_6bit_64simdWidth(in, out); - break; - case 7: - pack_7bit_64simdWidth(in, out); - break; - case 8: - pack_8bit_64simdWidth(in, out); - break; - case 9: - pack_9bit_64simdWidth(in, out); - break; - case 10: - pack_10bit_64simdWidth(in, out); - break; - case 11: - pack_11bit_64simdWidth(in, out); - break; - case 12: - pack_12bit_64simdWidth(in, out); - break; - case 13: - pack_13bit_64simdWidth(in, out); - break; - case 14: - pack_14bit_64simdWidth(in, out); - break; - case 15: - pack_15bit_64simdWidth(in, out); - break; - case 16: - pack_16bit_64simdWidth(in, out); - break; - case 17: - pack_17bit_64simdWidth(in, out); - break; - case 18: - pack_18bit_64simdWidth(in, out); - break; - case 19: - pack_19bit_64simdWidth(in, out); - break; - case 20: - pack_20bit_64simdWidth(in, out); - break; - case 21: - pack_21bit_64simdWidth(in, out); - break; - case 22: - pack_22bit_64simdWidth(in, out); - break; - case 23: - pack_23bit_64simdWidth(in, out); - break; - case 24: - pack_24bit_64simdWidth(in, out); - break; - case 25: - pack_25bit_64simdWidth(in, out); - break; - case 26: - pack_26bit_64simdWidth(in, out); - break; - case 27: - pack_27bit_64simdWidth(in, out); - break; - case 28: - pack_28bit_64simdWidth(in, out); - break; - case 29: - pack_29bit_64simdWidth(in, out); - break; - case 30: - pack_30bit_64simdWidth(in, out); - break; - case 31: - pack_31bit_64simdWidth(in, out); - break; - case 32: - pack_32bit_64simdWidth(in, out); - break; - case 33: - pack_33bit_64simdWidth(in, out); - break; - case 34: - pack_34bit_64simdWidth(in, out); - break; - case 35: - pack_35bit_64simdWidth(in, out); - break; - case 36: - pack_36bit_64simdWidth(in, out); - break; - case 37: - pack_37bit_64simdWidth(in, out); - break; - case 38: - pack_38bit_64simdWidth(in, out); - break; - case 39: - pack_39bit_64simdWidth(in, out); - break; - case 40: - pack_40bit_64simdWidth(in, out); - break; - case 41: - pack_41bit_64simdWidth(in, out); - break; - case 42: - pack_42bit_64simdWidth(in, out); - break; - case 43: - pack_43bit_64simdWidth(in, out); - break; - case 44: - pack_44bit_64simdWidth(in, out); - break; - case 45: - pack_45bit_64simdWidth(in, out); - break; - case 46: - pack_46bit_64simdWidth(in, out); - break; - case 47: - pack_47bit_64simdWidth(in, out); - break; - case 48: - pack_48bit_64simdWidth(in, out); - break; - case 49: - pack_49bit_64simdWidth(in, out); - break; - case 50: - pack_50bit_64simdWidth(in, out); - break; - case 51: - pack_51bit_64simdWidth(in, out); - break; - case 52: - pack_52bit_64simdWidth(in, out); - break; - case 53: - pack_53bit_64simdWidth(in, out); - break; - case 54: - pack_54bit_64simdWidth(in, out); - break; - case 55: - pack_55bit_64simdWidth(in, out); - break; - case 56: - pack_56bit_64simdWidth(in, out); - break; - case 57: - pack_57bit_64simdWidth(in, out); - break; - case 58: - pack_58bit_64simdWidth(in, out); - break; - case 59: - pack_59bit_64simdWidth(in, out); - break; - case 60: - pack_60bit_64simdWidth(in, out); - break; - case 61: - pack_61bit_64simdWidth(in, out); - break; - case 62: - pack_62bit_64simdWidth(in, out); - break; - case 63: - pack_63bit_64simdWidth(in, out); - break; - case 64: - pack_64bit_64simdWidth(in, out); - break; - } -} -}}; // namespace fastlanes::bitpack diff --git a/src/cor/prm/bitpack/bitpack_decompress.cpp b/src/cor/prm/bitpack/bitpack_decompress.cpp deleted file mode 100644 index 10ff755b..00000000 --- a/src/cor/prm/bitpack/bitpack_decompress.cpp +++ /dev/null @@ -1,17303 +0,0 @@ -#include "fls/cor/prm/bitpack.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t - -namespace fastlanes { namespace bitpack { -static void unpack_1bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_2bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_3bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_4bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_5bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_6bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_7bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_8bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 128); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 256); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 384); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 512); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 640); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 768); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 896); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = register_0; - } -} -static void unpack_1bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_2bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_3bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_4bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_5bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_6bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_7bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_8bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_9bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_10bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_11bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_12bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_13bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_14bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_15bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_16bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 128); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 256); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 320); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 384); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 448); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 512); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 576); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 640); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 704); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 768); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 832); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 896); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 960); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = register_0; - } -} -static void unpack_1bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_2bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_3bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_4bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_5bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_6bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_7bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_8bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_9bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_10bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_11bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_12bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_13bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_14bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_15bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_16bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_17bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_18bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_19bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_20bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_21bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_22bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_23bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_24bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_25bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_26bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_27bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_28bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_29bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_30bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_31bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_32bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 128); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 256); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 320); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 352); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 384); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 416); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 448); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 512); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 544); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 576); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 608); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 640); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 672); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 704); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 736); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 768); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 800); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 832); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 864); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 896); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 928); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 960); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 992); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = register_0; - } -} -static void unpack_1bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_2bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_3bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_4bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_5bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_6bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_7bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_8bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_9bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_10bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_11bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_12bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_13bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_14bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_15bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_16bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_17bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_18bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_19bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_20bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_21bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_22bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_23bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_24bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_25bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_26bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_27bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_28bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_29bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_30bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_31bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_32bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_33bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_34bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 = (register_0) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_35bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_36bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_37bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_38bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 = (register_0) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_39bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_40bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_41bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_42bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_43bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_44bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_45bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_46bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 = (register_0) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_47bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_48bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_49bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_50bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 = (register_0) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_51bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_52bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_53bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_54bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_55bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_56bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_57bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_58bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 = (register_0) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_59bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 57) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 57; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_60bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_61bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 57; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 59) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 59) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 59; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 57) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_62bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 = (register_0) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 976); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_63bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 63) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 62) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 61) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 59) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 57) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 57; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 59) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 59; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 61) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 976); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 61; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 62) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 992); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 62; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 63) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_64bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 128); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 256); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 272); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 304); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 320); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 352); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 368); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 384); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 416); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 448); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 464); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 496); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 512); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 528); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 544); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 560); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 576); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 592); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 608); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 624); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 640); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 656); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 672); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 688); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 704); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 720); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 736); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 752); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 768); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 784); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 800); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 816); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 832); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 848); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 864); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 880); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 896); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 912); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 928); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 944); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 960); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 976); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 992); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 1008); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = register_0; - } -} - -void Decompress08(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint8_t* __restrict in = 0; - uint8_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_8ow_8crw_1uf(in, out); - break; - case 2: - unpack_2bw_8ow_8crw_1uf(in, out); - break; - case 3: - unpack_3bw_8ow_8crw_1uf(in, out); - break; - case 4: - unpack_4bw_8ow_8crw_1uf(in, out); - break; - case 5: - unpack_5bw_8ow_8crw_1uf(in, out); - break; - case 6: - unpack_6bw_8ow_8crw_1uf(in, out); - break; - case 7: - unpack_7bw_8ow_8crw_1uf(in, out); - break; - case 8: - unpack_8bw_8ow_8crw_1uf(in, out); - break; - } -} -void Decompress16(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint16_t* __restrict in = 0; - uint16_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_16ow_16crw_1uf(in, out); - break; - case 2: - unpack_2bw_16ow_16crw_1uf(in, out); - break; - case 3: - unpack_3bw_16ow_16crw_1uf(in, out); - break; - case 4: - unpack_4bw_16ow_16crw_1uf(in, out); - break; - case 5: - unpack_5bw_16ow_16crw_1uf(in, out); - break; - case 6: - unpack_6bw_16ow_16crw_1uf(in, out); - break; - case 7: - unpack_7bw_16ow_16crw_1uf(in, out); - break; - case 8: - unpack_8bw_16ow_16crw_1uf(in, out); - break; - case 9: - unpack_9bw_16ow_16crw_1uf(in, out); - break; - case 10: - unpack_10bw_16ow_16crw_1uf(in, out); - break; - case 11: - unpack_11bw_16ow_16crw_1uf(in, out); - break; - case 12: - unpack_12bw_16ow_16crw_1uf(in, out); - break; - case 13: - unpack_13bw_16ow_16crw_1uf(in, out); - break; - case 14: - unpack_14bw_16ow_16crw_1uf(in, out); - break; - case 15: - unpack_15bw_16ow_16crw_1uf(in, out); - break; - case 16: - unpack_16bw_16ow_16crw_1uf(in, out); - break; - } -} - -void Decompress32(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint32_t* __restrict in = 0; - uint32_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_32ow_32crw_1uf(in, out); - break; - case 2: - unpack_2bw_32ow_32crw_1uf(in, out); - break; - case 3: - unpack_3bw_32ow_32crw_1uf(in, out); - break; - case 4: - unpack_4bw_32ow_32crw_1uf(in, out); - break; - case 5: - unpack_5bw_32ow_32crw_1uf(in, out); - break; - case 6: - unpack_6bw_32ow_32crw_1uf(in, out); - break; - case 7: - unpack_7bw_32ow_32crw_1uf(in, out); - break; - case 8: - unpack_8bw_32ow_32crw_1uf(in, out); - break; - case 9: - unpack_9bw_32ow_32crw_1uf(in, out); - break; - case 10: - unpack_10bw_32ow_32crw_1uf(in, out); - break; - case 11: - unpack_11bw_32ow_32crw_1uf(in, out); - break; - case 12: - unpack_12bw_32ow_32crw_1uf(in, out); - break; - case 13: - unpack_13bw_32ow_32crw_1uf(in, out); - break; - case 14: - unpack_14bw_32ow_32crw_1uf(in, out); - break; - case 15: - unpack_15bw_32ow_32crw_1uf(in, out); - break; - case 16: - unpack_16bw_32ow_32crw_1uf(in, out); - break; - case 17: - unpack_17bw_32ow_32crw_1uf(in, out); - break; - case 18: - unpack_18bw_32ow_32crw_1uf(in, out); - break; - case 19: - unpack_19bw_32ow_32crw_1uf(in, out); - break; - case 20: - unpack_20bw_32ow_32crw_1uf(in, out); - break; - case 21: - unpack_21bw_32ow_32crw_1uf(in, out); - break; - case 22: - unpack_22bw_32ow_32crw_1uf(in, out); - break; - case 23: - unpack_23bw_32ow_32crw_1uf(in, out); - break; - case 24: - unpack_24bw_32ow_32crw_1uf(in, out); - break; - case 25: - unpack_25bw_32ow_32crw_1uf(in, out); - break; - case 26: - unpack_26bw_32ow_32crw_1uf(in, out); - break; - case 27: - unpack_27bw_32ow_32crw_1uf(in, out); - break; - case 28: - unpack_28bw_32ow_32crw_1uf(in, out); - break; - case 29: - unpack_29bw_32ow_32crw_1uf(in, out); - break; - case 30: - unpack_30bw_32ow_32crw_1uf(in, out); - break; - case 31: - unpack_31bw_32ow_32crw_1uf(in, out); - break; - case 32: - unpack_32bw_32ow_32crw_1uf(in, out); - break; - } -} -void Decompress64(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint64_t* __restrict in = 0; - uint64_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_64ow_64crw_1uf(in, out); - break; - case 2: - unpack_2bw_64ow_64crw_1uf(in, out); - break; - case 3: - unpack_3bw_64ow_64crw_1uf(in, out); - break; - case 4: - unpack_4bw_64ow_64crw_1uf(in, out); - break; - case 5: - unpack_5bw_64ow_64crw_1uf(in, out); - break; - case 6: - unpack_6bw_64ow_64crw_1uf(in, out); - break; - case 7: - unpack_7bw_64ow_64crw_1uf(in, out); - break; - case 8: - unpack_8bw_64ow_64crw_1uf(in, out); - break; - case 9: - unpack_9bw_64ow_64crw_1uf(in, out); - break; - case 10: - unpack_10bw_64ow_64crw_1uf(in, out); - break; - case 11: - unpack_11bw_64ow_64crw_1uf(in, out); - break; - case 12: - unpack_12bw_64ow_64crw_1uf(in, out); - break; - case 13: - unpack_13bw_64ow_64crw_1uf(in, out); - break; - case 14: - unpack_14bw_64ow_64crw_1uf(in, out); - break; - case 15: - unpack_15bw_64ow_64crw_1uf(in, out); - break; - case 16: - unpack_16bw_64ow_64crw_1uf(in, out); - break; - case 17: - unpack_17bw_64ow_64crw_1uf(in, out); - break; - case 18: - unpack_18bw_64ow_64crw_1uf(in, out); - break; - case 19: - unpack_19bw_64ow_64crw_1uf(in, out); - break; - case 20: - unpack_20bw_64ow_64crw_1uf(in, out); - break; - case 21: - unpack_21bw_64ow_64crw_1uf(in, out); - break; - case 22: - unpack_22bw_64ow_64crw_1uf(in, out); - break; - case 23: - unpack_23bw_64ow_64crw_1uf(in, out); - break; - case 24: - unpack_24bw_64ow_64crw_1uf(in, out); - break; - case 25: - unpack_25bw_64ow_64crw_1uf(in, out); - break; - case 26: - unpack_26bw_64ow_64crw_1uf(in, out); - break; - case 27: - unpack_27bw_64ow_64crw_1uf(in, out); - break; - case 28: - unpack_28bw_64ow_64crw_1uf(in, out); - break; - case 29: - unpack_29bw_64ow_64crw_1uf(in, out); - break; - case 30: - unpack_30bw_64ow_64crw_1uf(in, out); - break; - case 31: - unpack_31bw_64ow_64crw_1uf(in, out); - break; - case 32: - unpack_32bw_64ow_64crw_1uf(in, out); - break; - case 33: - unpack_33bw_64ow_64crw_1uf(in, out); - break; - case 34: - unpack_34bw_64ow_64crw_1uf(in, out); - break; - case 35: - unpack_35bw_64ow_64crw_1uf(in, out); - break; - case 36: - unpack_36bw_64ow_64crw_1uf(in, out); - break; - case 37: - unpack_37bw_64ow_64crw_1uf(in, out); - break; - case 38: - unpack_38bw_64ow_64crw_1uf(in, out); - break; - case 39: - unpack_39bw_64ow_64crw_1uf(in, out); - break; - case 40: - unpack_40bw_64ow_64crw_1uf(in, out); - break; - case 41: - unpack_41bw_64ow_64crw_1uf(in, out); - break; - case 42: - unpack_42bw_64ow_64crw_1uf(in, out); - break; - case 43: - unpack_43bw_64ow_64crw_1uf(in, out); - break; - case 44: - unpack_44bw_64ow_64crw_1uf(in, out); - break; - case 45: - unpack_45bw_64ow_64crw_1uf(in, out); - break; - case 46: - unpack_46bw_64ow_64crw_1uf(in, out); - break; - case 47: - unpack_47bw_64ow_64crw_1uf(in, out); - break; - case 48: - unpack_48bw_64ow_64crw_1uf(in, out); - break; - case 49: - unpack_49bw_64ow_64crw_1uf(in, out); - break; - case 50: - unpack_50bw_64ow_64crw_1uf(in, out); - break; - case 51: - unpack_51bw_64ow_64crw_1uf(in, out); - break; - case 52: - unpack_52bw_64ow_64crw_1uf(in, out); - break; - case 53: - unpack_53bw_64ow_64crw_1uf(in, out); - break; - case 54: - unpack_54bw_64ow_64crw_1uf(in, out); - break; - case 55: - unpack_55bw_64ow_64crw_1uf(in, out); - break; - case 56: - unpack_56bw_64ow_64crw_1uf(in, out); - break; - case 57: - unpack_57bw_64ow_64crw_1uf(in, out); - break; - case 58: - unpack_58bw_64ow_64crw_1uf(in, out); - break; - case 59: - unpack_59bw_64ow_64crw_1uf(in, out); - break; - case 60: - unpack_60bw_64ow_64crw_1uf(in, out); - break; - case 61: - unpack_61bw_64ow_64crw_1uf(in, out); - break; - case 62: - unpack_62bw_64ow_64crw_1uf(in, out); - break; - case 63: - unpack_63bw_64ow_64crw_1uf(in, out); - break; - case 64: - unpack_64bw_64ow_64crw_1uf(in, out); - break; - } -} -}}; // namespace fastlanes::bitpack diff --git a/src/cor/prm/factor/CMakeLists.txt b/src/cor/prm/factor/CMakeLists.txt deleted file mode 100644 index b67390a3..00000000 --- a/src/cor/prm/factor/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_factor_prm - OBJECT - decompress.cpp - compress.cpp) - -target_compile_definitions(fls_factor_prm PRIVATE IS_SCALAR) -target_compile_options(fls_factor_prm PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_factor_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_factor_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/factor/compress.cpp b/src/cor/prm/factor/compress.cpp deleted file mode 100644 index c7a9524b..00000000 --- a/src/cor/prm/factor/compress.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include "fls/cor/prm/factor.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t -#include // for __restrict - -namespace fastlanes { namespace factor { -void Compress64(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - /* */ /* Reinterpret. */ - /* -auto *factor_col = reinterpret_cast(state.col1); -auto src_vec = reinterpret_cast(src); -auto factor = factor_col[0]; - -*/ /* Compute. */ /* - for (size_t i = 0; i < 1024; ++i) { - src_vec[i] = src_vec[i] - factor; - } - - */ /* State. */ /* -// state.col1 += 8;*/ -} -}} // namespace fastlanes::factor diff --git a/src/cor/prm/factor/decompress.cpp b/src/cor/prm/factor/decompress.cpp deleted file mode 100644 index 288ac182..00000000 --- a/src/cor/prm/factor/decompress.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include "fls/cor/prm/factor.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t -#include // for __restrict - -namespace fastlanes { namespace factor { -}} diff --git a/src/cor/prm/ffor/CMakeLists.txt b/src/cor/prm/ffor/CMakeLists.txt deleted file mode 100644 index b950a812..00000000 --- a/src/cor/prm/ffor/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_library(fls_ffor_prm - OBJECT - ffor_decompress.cpp - ffor_compress.cpp - ffor_analyze.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_ffor_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_ffor_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/ffor/ffor_analyze.cpp b/src/cor/prm/ffor/ffor_analyze.cpp deleted file mode 100644 index cb371b25..00000000 --- a/src/cor/prm/ffor/ffor_analyze.cpp +++ /dev/null @@ -1,202 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/mtd.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/stt/histogram.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include -#include -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -template -bool ffor_prm::is_exception(Params params, T val) { - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - FLS_ASSERT_CORRECT_BW(params.bw) - FLS_ASSERT_CORRECT_N(params.exc_c) - FLS_ASSERT_CORRECT_SZ(params.size()) - // [fixme] - if (params.bw == TYPE_BIT::VALUE) { - return false; - } - - T lower_bound = static_cast(params.base); - - if (val < lower_bound) { - return true; - } - if (val - lower_bound > MAX_BIT(params.bw)) { - return true; - } - return false; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return false; - } -} - -template -Params ffor_prm::find_best_params(Histogram& histogram, n_t idx) { - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - - /* Initialize */ - Params best, next; - auto& val_vec = histogram.val_vec; - auto& rep_vec = histogram.rep_vec; - const T lower_bound = val_vec[idx]; - T upper_bound = lower_bound; - - /* best = worst */ - best = {TYPE_BIT::VALUE, 0, 0}; - - // can we do better? - for (int i = TYPE_BIT::VALUE - 1; i >= 0; --i) { - /* Compute the upperbound*/ - upper_bound = SafeUpperBound(lower_bound, i); - - /* Compute the next setting*/ - next = { - static_cast(i), - lower_bound, - static_cast(typed_patch::count_exceptions(lower_bound, upper_bound, val_vec, rep_vec))}; - - FLS_ASSERT_CORRECT_EXC_C(next.exc_c) - /* Check to see if there is a better setting. */ - if (next.size() < best.size() & next.exc_c <= CFG::PATCH::EXC_LIMIT_C) { - best = next; - } - } - - return best; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return Params(); - } -} - -template -void ffor_analyze(Vec& src_vec, Vec& des_vec, CompressState& stt) { - /* copy data into des.*/ - memcpy( - des_vec.buf_arr[stt.cur_des_buff + 1].mutable_data(), src_vec.buf_arr[stt.cur_src_buff].data(), vec_bsz()); - - auto min = std::numeric_limits::max(); - auto max = std::numeric_limits::min(); - - auto* in_p = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 1].data()); - for (size_t i {0}; i < 1024; ++i) { - if (in_p[i] < min) { - min = in_p[i]; - } - if (in_p[i] > max) { - max = in_p[i]; - } - } - - uint64_t delta = (static_cast(max) - static_cast(min)); - auto bits_per_digit = ceil(log2(delta + 1)); - uint8_t bw = bits_per_digit; - - stt.bw = bw; - std::memcpy(stt.base, &min, sizeof(T)); - // base_p[0] = min; - FLS_PLOG_KEY_VALUE("bw", std::to_string(stt.bw)) - FLS_PLOG_KEY_VALUE("min", std::to_string(min)) - FLS_PLOG_KEY_VALUE("max", std::to_string(max)) -} - -template -void ffor_patch_analyze(Vec& src_vec, Vec& des_vec, CompressState& stt) { - /* copy data into stt_buf.*/ - Params best, next; - memcpy(des_vec.buf_arr[1].mutable_data(), src_vec.buf_arr[0].data(), vec_bsz()); - - /* [op]: the tmp buf is sorted already, the minmax is the first and last element. */ - /* [op]: move to stt to reduce allocation. */ - auto histo = Histogram::cal(des_vec.buf_arr[1].mutable_data()); - auto min_max = MinMax::cal(des_vec.buf_arr[1].mutable_data()); - n_t base_c = histo->rep_vec.size(); - - best = ffor_prm::find_best_params(*histo, 0); - - /* Compute. */ - for (size_t i {1}; i < base_c; ++i) { - next = ffor_prm::find_best_params(*histo, i); - - if (next.size() < best.size()) { - best = next; - } - } - - stt.bw = best.bw; - std::memcpy(stt.base, &best.base, sizeof(T)); - FLS_ASSERT_CORRECT_BASE_UB(best.base, min_max->max) - - stt.exc_c = best.exc_c; - - /* Add exception positions . */ - uint16_t exc_c {0}; - - for (size_t i {0}; i < vec_n_tup(); ++i) { - T& val = reinterpret_cast(src_vec.buf_arr[0].mutable_data())[i]; - if (ffor_prm::is_exception(best, val)) { - stt.exc_pos_arr[exc_c++] = i; - } - } - - FLS_PLOG_KEY_VALUE("VEC", ToStr::to_str(reinterpret_cast(src_vec.buf_arr[0].mutable_data()))) - FLS_PLOG_KEY_VALUE("BW", std::to_string(stt.bw)) - FLS_PLOG_KEY_VALUE("EXC_C", ToStr::to_str(exc_c)) - FLS_PLOG_KEY_VALUE("BASE", ToStr::to_hex(stt.base, sizeof(T))); -} - -template -cmpr_fun_t ffor_prm::ResolveAnalyzeFunc() { - if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; - } -} - -template -cmpr_fun_t ffor_prm::ResolvePatchAnalyzeFunc() { - if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; - } -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(ffor_prm) -} // namespace fastlanes diff --git a/src/cor/prm/ffor/ffor_compress.cpp b/src/cor/prm/ffor/ffor_compress.cpp deleted file mode 100644 index 67cc4041..00000000 --- a/src/cor/prm/ffor/ffor_compress.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/ffor.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" - -namespace fastlanes { -template -static void ffor_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* base_p = reinterpret_cast(stt.base); - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - auto out_p = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 2].mutable_data()); - - generated::ffor::fallback::scalar::ffor(in_arr, out_p, stt.bw, base_p); - - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&stt.bw, 1); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(stt.base, sizeof(T)); - des_vec.buf_arr[stt.cur_des_buff + 2].UnsafeAdvance(ffor_prm::vec_sz(stt.bw)); - - FLS_PLOG_KEY_VALUE("VEC", ToStr::to_str(const_cast(in_arr))) - FLS_PLOG_KEY_VALUE("BW", std::to_string(stt.bw)) - FLS_PLOG_KEY_VALUE("BASE", ToStr::to_hex(stt.base, sizeof(T))); - - stt.cur_des_buff += 3; -} - -template -cmpr_fun_t ffor_prm::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -template -n_t ffor_prm::vec_sz(uint8_t bw) { - FLS_ASSERT_CORRECT_BW(bw) - - return bw * vec_sz_1bit(); -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(ffor_prm) - -} // namespace fastlanes diff --git a/src/cor/prm/ffor/ffor_decompress.cpp b/src/cor/prm/ffor/ffor_decompress.cpp deleted file mode 100644 index bff14c4a..00000000 --- a/src/cor/prm/ffor/ffor_decompress.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls_gen/unffor/unffor.hpp" -#include - -namespace fastlanes { -template -static void ffor_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 2]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 1]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 0]) - - uint8_t bw = *src.arr[stt.cur_src_arr - 2]; - auto* base_p = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* in_p = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr]); - - generated::unffor::fallback::scalar::unffor(in_p, out_p, bw, base_p); - - FLS_PLOG_KEY_VALUE("VEC", ToStr::to_str(reinterpret_cast(out_p))) - FLS_PLOG_KEY_VALUE("BW", std::to_string(bw)) - FLS_PLOG_KEY_VALUE("BASE", ToStr::to_hex(stt.base, sizeof(T))); - - stt.cur_src_arr -= 3; - std::memcpy(stt.base, base_p, sizeof(T)); -} - -template -de_cmpr_fun_t ffor_prm::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(ffor_prm) -} // namespace fastlanes diff --git a/src/cor/prm/mem_cpy/CMakeLists.txt b/src/cor/prm/mem_cpy/CMakeLists.txt deleted file mode 100644 index f6a68144..00000000 --- a/src/cor/prm/mem_cpy/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_mem_cpy_exp - OBJECT - mem_cpy_decompress.cpp - mem_cpy_compress.cpp) - -target_compile_definitions(fls_mem_cpy_exp PRIVATE IS_SCALAR) -target_compile_options(fls_mem_cpy_exp PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(fls_mem_cpy_exp PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_mem_cpy_exp PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/mem_cpy/mem_cpy_compress.cpp b/src/cor/prm/mem_cpy/mem_cpy_compress.cpp deleted file mode 100644 index 4bf696f3..00000000 --- a/src/cor/prm/mem_cpy/mem_cpy_compress.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_cpy_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" - -namespace fastlanes { -template -static void mem_cpy_cmpr(Vec& src_vec, Vec& des_vec, CompressState& stt) { - T* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - des_vec.buf_arr[stt.cur_des_buff].UnsafeAppend(in_arr, src_vec.buf_arr[stt.cur_src_buff].length()); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(in_arr)); - - stt.cur_des_buff += 1; - stt.cur_src_buff += 1; -} - -static void off_mem_cpy_cmpr(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = src_vec.buf_arr[stt.cur_src_buff].mutable_data(); - - /* in smart offset array the last tuple show the number of bytes. */ - uint32_t byte_c = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 1].mutable_data())[1023]; - // FLS_ASSERT_LE(byte_c, CFG::BUF::SZ) // todo[buf] - - des_vec.buf_arr[stt.cur_des_buff].UnsafeAppend(in_arr, byte_c); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(in_arr)); - - stt.cur_des_buff += 1; - stt.cur_src_buff += 1; -} - -template -cmpr_fun_t mem_cpy::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return off_mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } - - FLS_ABORT("IT IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, mem_cpy::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/mem_cpy/mem_cpy_decompress.cpp b/src/cor/prm/mem_cpy/mem_cpy_decompress.cpp deleted file mode 100644 index ee85e1c7..00000000 --- a/src/cor/prm/mem_cpy/mem_cpy_decompress.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_cpy_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include - -namespace fastlanes { -template -static void mem_cpy_decmpr(PageParam src, VecParam des, DecompressState& stt) { - auto* in_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); - - std::memcpy(out_arr, in_arr, vec_bsz()); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(out_arr)); - - stt.cur_des_arr -= 1; - stt.cur_src_arr -= 1; -} - -static void off_mem_cpy_decmpr(PageParam src, VecParam des, DecompressState& stt) { - auto* in_arr = src.arr[stt.cur_src_arr - 0]; - auto* out_arr = des.arr[stt.cur_des_arr - 0]; - - /* in smart offset array the last tuple show the number of bytes. */ - uint32_t byte_c = reinterpret_cast(des.arr[stt.cur_des_arr + 1])[1023]; - // FLS_ASSERT_LE(byte_c, CFG::BUF::SZ) // todo[buf] - std::memcpy(out_arr, in_arr, byte_c); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(out_arr)); - - stt.cur_des_arr -= 1; - stt.cur_src_arr -= 1; -} - -template -de_cmpr_fun_t mem_cpy::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return off_mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } - - FLS_ABORT("IT IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, mem_cpy::ResolveDecompressFunc) -} // namespace fastlanes diff --git a/src/cor/prm/mem_trs/CMakeLists.txt b/src/cor/prm/mem_trs/CMakeLists.txt deleted file mode 100644 index c6ca11d7..00000000 --- a/src/cor/prm/mem_trs/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_mem_trs_prm - OBJECT - mem_trs_decompress.cpp - mem_trs_compress.cpp) - -target_compile_definitions(fls_mem_trs_prm PRIVATE IS_SCALAR) -target_compile_options(fls_mem_trs_prm PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(fls_mem_trs_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_mem_trs_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/mem_trs/mem_trs_compress.cpp b/src/cor/prm/mem_trs/mem_trs_compress.cpp deleted file mode 100644 index 26eb4ad8..00000000 --- a/src/cor/prm/mem_trs/mem_trs_compress.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_trs.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/transpose/transpose.hpp" -#include - -namespace fastlanes { -static void Compress64(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto size = src_vec.buf_arr[stt.cur_src_buff].length(); - const uint8_t* in_p = src_vec.buf_arr[stt.cur_src_buff++].mutable_data(); - - /* Fixme: not efficient. could be easily solved by a better local buffer anf memory pool manager */ - uint8_t tmp[1024 * 8]; - generated::transpose::fallback::scalar::transpose_i(reinterpret_cast(in_p), - reinterpret_cast(tmp)); - - des_vec.buf_arr[stt.cur_des_buff++].UnsafeAppend(tmp, size); -} - -static void Compress32(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto size = src_vec.buf_arr[stt.cur_src_buff].length(); - const uint8_t* in_p = src_vec.buf_arr[stt.cur_src_buff++].mutable_data(); - - uint8_t tmp[1024 * 8]; - generated::transpose::fallback::scalar::transpose_i(reinterpret_cast(in_p), - reinterpret_cast(tmp)); - - des_vec.buf_arr[stt.cur_des_buff++].UnsafeAppend(tmp, size); -} - -template -cmpr_fun_t mem_trs::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return Compress64; - } else if constexpr (std::is_same()) { - return Compress64; - } else if constexpr (std::is_same()) { - return Compress32; - } else if constexpr (std::is_same()) { - return Compress32; - } - - FLS_ABORT("IT IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, mem_trs::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/mem_trs/mem_trs_decompress.cpp b/src/cor/prm/mem_trs/mem_trs_decompress.cpp deleted file mode 100644 index 7908450e..00000000 --- a/src/cor/prm/mem_trs/mem_trs_decompress.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_trs.hpp" -#include "fls_gen/untranspose/untranspose.hpp" - -namespace fastlanes { -static void Decompress64(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr]) - - generated::untranspose::fallback::scalar::untranspose_i(reinterpret_cast(src.arr[stt.cur_src_arr++]), - reinterpret_cast(des.arr[stt.cur_des_arr++])); -} - -static void Decompress32(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr]) - - generated::untranspose::fallback::scalar::untranspose_i(reinterpret_cast(src.arr[stt.cur_src_arr++]), - reinterpret_cast(des.arr[stt.cur_des_arr++])); -} - -template -de_cmpr_fun_t mem_trs::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return Decompress64; - } - if constexpr (std::is_same()) { - return Decompress64; - } - if constexpr (std::is_same()) { - return Decompress32; - } - if constexpr (std::is_same()) { - return Decompress32; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, mem_trs::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/off_cpy/CMakeLists.txt b/src/cor/prm/off_cpy/CMakeLists.txt deleted file mode 100644 index 8b137891..00000000 --- a/src/cor/prm/off_cpy/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/cor/prm/off_cpy/off_cpy_compress.cpp b/src/cor/prm/off_cpy/off_cpy_compress.cpp deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cor/prm/off_cpy/off_cpy_decompress.cpp b/src/cor/prm/off_cpy/off_cpy_decompress.cpp deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cor/prm/patch/CMakeLists.txt b/src/cor/prm/patch/CMakeLists.txt deleted file mode 100644 index 6139dfc8..00000000 --- a/src/cor/prm/patch/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -add_subdirectory(b_patch) -add_subdirectory(ll_patch) -add_subdirectory(s_patch) - -add_library(patch_prm - OBJECT - patch.cpp) - -target_compile_definitions(patch_prm PRIVATE IS_SCALAR) -target_compile_options(patch_prm PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/b_patch/CMakeLists.txt b/src/cor/prm/patch/b_patch/CMakeLists.txt deleted file mode 100644 index 10c8be66..00000000 --- a/src/cor/prm/patch/b_patch/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_b_patch_prm - OBJECT - b_patch_decompress.cpp - b_patch_compress.cpp) - -target_compile_definitions(fls_b_patch_prm PRIVATE IS_SCALAR) -target_compile_options(fls_b_patch_prm PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_b_patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_b_patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/b_patch/b_patch_compress.cpp b/src/cor/prm/patch/b_patch/b_patch_compress.cpp deleted file mode 100644 index ed587994..00000000 --- a/src/cor/prm/patch/b_patch/b_patch_compress.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/b_patch.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" - -namespace fastlanes { -template -static void b_patch_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - patch::b_patch_compress(stt.exc_pos_arr, stt.exc_c, stt.bitmap); - FLS_PLOG_KEY_VALUE("bitmap", ToStr::to_str(reinterpret_cast(stt.bitmap))); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.bitmap, CFG::BitMap::SZ); - - for (n_t i {0}; i < stt.exc_c; i++) { - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&in_arr[stt.exc_pos_arr[i]], sizeof(T)); - } - - stt.cur_des_buff += 2; - FLS_PLOG_KEY_VALUE("bitmap", ToStr::to_str(reinterpret_cast(stt.bitmap))); -} - -template -cmpr_fun_t b_patch::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return b_patch_compress; - } - if constexpr (std::is_same()) { - return b_patch_compress; - } - if constexpr (std::is_same()) { - return b_patch_compress; - } - if constexpr (std::is_same()) { - return b_patch_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, b_patch::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/b_patch/b_patch_decompress.cpp b/src/cor/prm/patch/b_patch/b_patch_decompress.cpp deleted file mode 100644 index 90009d71..00000000 --- a/src/cor/prm/patch/b_patch/b_patch_decompress.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/b_patch.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void b_patch_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - - auto* exc_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* bitmap_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr]); - - FLS_PLOG_KEY_VALUE("out_p", ToStr::to_str(out_p)); - - typed_patch::b_patch_decompress(out_p, bitmap_arr, exc_arr); - - stt.cur_src_arr -= 2; - - FLS_PLOG_KEY_VALUE("out_p", ToStr::to_str(out_p)); - FLS_PLOG_KEY_VALUE("bitmap", ToStr::to_str(reinterpret_cast(bitmap_arr))); -} - -template -de_cmpr_fun_t b_patch::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return b_patch_decompress; - } - if constexpr (std::is_same()) { - return b_patch_decompress; - } - if constexpr (std::is_same()) { - return b_patch_decompress; - } - if constexpr (std::is_same()) { - return b_patch_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, b_patch::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/ll_patch/CMakeLists.txt b/src/cor/prm/patch/ll_patch/CMakeLists.txt deleted file mode 100644 index d5e96458..00000000 --- a/src/cor/prm/patch/ll_patch/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_library(fls_ll_patch_prm - OBJECT - ll_patch_decompress.cpp - ll_patch_compress.cpp) - -target_compile_definitions(fls_ll_patch_prm PRIVATE IS_SCALAR) -target_compile_options(fls_ll_patch_prm PRIVATE "") - -if(ENABLE_IWYU) - set_target_properties(fls_ll_patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_ll_patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/ll_patch/ll_patch_compress.cpp b/src/cor/prm/patch/ll_patch/ll_patch_compress.cpp deleted file mode 100644 index bb61025c..00000000 --- a/src/cor/prm/patch/ll_patch/ll_patch_compress.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/ll_patch.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -template -static void ll_patch_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - const auto* base_p = reinterpret_cast(stt.base); - - rle_idx_t new_c {0}; - rle_idx_t old_c {0}; - //[fixme] - rle_idx_t tmp[vec_n_tup()]; - switch (stt.exc_c) { - case 0: - break; - default: - T curr_pos = stt.exc_pos_arr[old_c++]; - T next_pos = stt.exc_pos_arr[old_c++]; - T max_range = 1 << stt.bw; - - FLS_ASSERT_CORRECT_POS(curr_pos) - do { - tmp[new_c++] = curr_pos; - auto con = patch::is_compulsory_exc(curr_pos, next_pos, max_range); - if (old_c <= stt.exc_c && con) { - curr_pos += max_range; - } else { - curr_pos = next_pos; - next_pos = stt.exc_pos_arr[old_c++]; - } - } while (old_c <= stt.exc_c + 1); - stt.exc_c = new_c; - } - - /* push the number of exceptions. */ - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.exc_c, sizeof(rle_idx_t)); - /* push the pos of first exception. */ - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&tmp[0], sizeof(rle_idx_t)); - - /* push the exceptions and fix the linked list. */ - for (rle_idx_t i {0}; i < stt.exc_c; ++i) { - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&in_arr[tmp[i]], sizeof(T)); - - auto gap = tmp[i + 1] - tmp[i] + (*base_p) - 1; - in_arr[tmp[i]] = gap; - } - - FLS_PLOG_KEY_VALUE("after ll_patch", ToStr::to_str(in_arr)); - stt.cur_des_buff += 2; -} - -template -cmpr_fun_t ll_patch::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return ll_patch_compress; - } - if constexpr (std::is_same()) { - return ll_patch_compress; - } - if constexpr (std::is_same()) { - return ll_patch_compress; - } - if constexpr (std::is_same()) { - return ll_patch_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, ll_patch::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/ll_patch/ll_patch_decompress.cpp b/src/cor/prm/patch/ll_patch/ll_patch_decompress.cpp deleted file mode 100644 index 87f9acfe..00000000 --- a/src/cor/prm/patch/ll_patch/ll_patch_decompress.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/ll_patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void ll_patch_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - - auto* exc_p = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* exc_pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr]); - auto base = *(reinterpret_cast(stt.base)); - rle_idx_t exc_c = exc_pos_arr[0]; - - FLS_PLOG_KEY_VALUE("exe_c", std::to_string(exc_c)); - - rle_idx_t first_pos = exc_pos_arr[1]; - FLS_PLOG_KEY_VALUE("first_pos", std::to_string(first_pos)); - - FLS_PLOG_KEY_VALUE("out_arr", ToStr::to_str(out_p)); - FLS_PLOG_KEY_VALUE("exc_arr", ToStr::to_str(exc_p, exc_c)); - FLS_PLOG_KEY_VALUE("[exe_c, exc_first_pos] ", ToStr::to_str(exc_pos_arr, 2)); - - T cur = first_pos; - T offset {0}; - for (T i {0}; i < exc_c; ++i) { - offset = out_p[cur] - base + 1; - out_p[cur] = exc_p[i]; - cur += offset; - } - - stt.cur_src_arr -= 2; - - FLS_PLOG_KEY_VALUE("out_p", ToStr::to_str(out_p)); -} - -template -de_cmpr_fun_t ll_patch::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, ll_patch::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/patch.cpp b/src/cor/prm/patch/patch.cpp deleted file mode 100644 index c427e1cd..00000000 --- a/src/cor/prm/patch/patch.cpp +++ /dev/null @@ -1,152 +0,0 @@ -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/utl/bit_util.hpp" - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -template -bool typed_patch::is_exception(T lower_bound, T upper_bound, T val) { - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - if (val <= upper_bound && val >= lower_bound) { - return false; - } - return true; - } else { - FLS_ABORT("this is not supported!") - return false; - } -} - -void patch::b_patch_compress(rle_idx_t arr[], rle_idx_t c, bitmap_t bitmap) { - for (n_t i {0}; i < CFG::BitMap::UNIT_C; ++i) { - bitmap[i] = 0; - } - - for (auto i {0}; i < c; ++i) { - rle_idx_t unit_num = arr[i] / CFG::BitMap::UNIT_BIT; // Which unit? - rle_idx_t relative_idx = arr[i] % CFG::BitMap::UNIT_BIT; // Where in the unit? - bit::set(bitmap[unit_num], relative_idx); - } -} - -bool patch::is_compulsory_exc(rle_idx_t curr, rle_idx_t next, uint64_t range) { - FLS_ASSERT_CORRECT_RANGE(range) - - return static_cast(next - curr) > range; -} - -template -n_t typed_patch::count_exceptions(const T lower_bound, - const T upper_bound, - const std::vector& val_vec, - const std::vector& rep_vec) { - - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - FLS_ASSERT(!val_vec.empty(), "an empty vec", " "); - FLS_ASSERT(!rep_vec.empty(), "an empty vec", " "); - - n_t result {0}; - bool is_exc {false}; - - for (n_t i = 0; i < val_vec.size(); ++i) { - is_exc = is_exception(lower_bound, upper_bound, val_vec[i]); - if (!is_exc) { - continue; - } - /* It is an exception. - * Increase the number of exception by the repetition of this value. - */ - result += rep_vec[i]; - } - - return result; - } else { - FLS_ABORT("this is not supported!") - return 0; - } -} - -template -void typed_patch::b_patch_decompress(T* out_p, uint64_t* bitmap, T* exc_p) { - int c {0}; - unit_t unit; - for (size_t i {0}; i < CFG::BitMap::UNIT_C; ++i) { - unit = bitmap[i]; - while (unit != 0) { - uint64_t t = unit & -unit; - int r = __builtin_ctzll(unit); - out_p[i * 64 + r] = exc_p[c++]; - unit ^= t; - } - } -} - -template -void typed_patch::l_patch_decompress(T* in, rle_idx_t first, rle_idx_t c) { -} - -template -void typed_patch::l_patch_compress(T* in, T* exc, rle_idx_t arr[], rle_idx_t c, n_t max_range) { - // if (stt.exc_c == 0) { - // /* push the pos of first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); - // /* push the number of exceptions */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); - // - // stt.cur_des_arr += 2; - // - // return; - // } - // - // if (stt.exc_c == 1) { - // /* push the pos of first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_pos_arr, sizeof(pos_t)); - // /* push the number of exceptions */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); - // /* push the first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(exc_arr, sizeof(T)); - // = - // stt.cur_des_arr += 2; - // - // return; - // } - // - // /* push the pos of first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_pos_arr[0], sizeof(pos_t)); - // /* push the first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(exc_arr + 0, sizeof(T)); - // - // T curr_pos = stt.exc_pos_arr[0]; - // T next_pos = stt.exc_pos_arr[1]; - // T max_range = 1 << stt.bw; - // pos_t new_c {0}; - // for (c_t i {1}; i < stt.exc_c; ++i) { - // curr_pos = stt.exc_pos_arr[i - 1]; - // next_pos = stt.exc_pos_arr[i]; - // while (patch::is_compulsory_exc(curr_pos, next_pos, max_range)) { - // /* Insert. */ - // in_arr[curr_pos] = max_range + (*base_p) - 1; - // curr_pos += max_range; - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(in_arr + curr_pos, sizeof(T)); - // new_c++; - // } - // in_arr[curr_pos] = (next_pos - curr_pos) + (*base_p) - 1; - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(exc_arr + i, sizeof(T)); - // } - // in_arr[next_pos] = *(base_p); - // - // /* forth, push the number of exceptions */ - // stt.exc_c += new_c; - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(typed_patch) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/s_patch/CMakeLists.txt b/src/cor/prm/patch/s_patch/CMakeLists.txt deleted file mode 100644 index 461f49be..00000000 --- a/src/cor/prm/patch/s_patch/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_s_patch_prm - OBJECT - s_patch_decompress.cpp - s_patch_compress.cpp) - -target_compile_definitions(fls_s_patch_prm PRIVATE IS_SCALAR) -target_compile_options(fls_s_patch_prm PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_s_patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_s_patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/s_patch/s_patch_compress.cpp b/src/cor/prm/patch/s_patch/s_patch_compress.cpp deleted file mode 100644 index 11d3a528..00000000 --- a/src/cor/prm/patch/s_patch/s_patch_compress.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/s_patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void s_patch_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - /* first, push the number of exceptions */ - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.exc_c, sizeof(uint16_t)); - - for (n_t i {0}; i < stt.exc_c; i++) { - /* [TODO], after recursive compression store bw bits of exception. */ - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&in_arr[stt.exc_pos_arr[i]], sizeof(T)); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.exc_pos_arr[i], sizeof(uint16_t)); - } - - stt.cur_des_buff += 2; -} - -template -cmpr_fun_t s_patch::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return s_patch_compress; - } - if constexpr (std::is_same()) { - return s_patch_compress; - } - if constexpr (std::is_same()) { - return s_patch_compress; - } - if constexpr (std::is_same()) { - return s_patch_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, s_patch::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/s_patch/s_patch_decompress.cpp b/src/cor/prm/patch/s_patch/s_patch_decompress.cpp deleted file mode 100644 index 0339ceb5..00000000 --- a/src/cor/prm/patch/s_patch/s_patch_decompress.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/s_patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void s_patch_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - - auto* exc_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* exc_pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_arr = reinterpret_cast(des.arr[stt.cur_des_arr]); - uint16_t exc_c = exc_pos_arr[0]; - - for (uint16_t i {0}; i < exc_c; ++i) { - auto next_pos = exc_pos_arr[i + 1]; - out_arr[next_pos] = exc_arr[i]; - } - - stt.cur_src_arr -= 2; - - FLS_PLOG_KEY_VALUE("exc_arr", ToStr::to_str(exc_arr, exc_c)); - FLS_PLOG_KEY_VALUE("after s_patch", ToStr::to_str(out_arr)); -} - -template -de_cmpr_fun_t s_patch::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return s_patch_decompress; - } - if constexpr (std::is_same()) { - return s_patch_decompress; - } - if constexpr (std::is_same()) { - return s_patch_decompress; - } - if constexpr (std::is_same()) { - return s_patch_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, s_patch::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rle/CMakeLists.txt b/src/cor/prm/rle/CMakeLists.txt deleted file mode 100644 index 81e2800f..00000000 --- a/src/cor/prm/rle/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_library(fls_rle_prm - OBJECT - rle_decompress.cpp - rle_compress.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_rle_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_rle_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/rle/rle_compress.cpp b/src/cor/prm/rle/rle_compress.cpp deleted file mode 100644 index 67d4c3dc..00000000 --- a/src/cor/prm/rle/rle_compress.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rle_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/transpose/transpose.hpp" - -namespace fastlanes { -template -static void rle_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - const auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 0].data()); - auto* pos_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 1].mutable_data()); - auto* tmp_pos_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 1].mutable_data()); - - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - auto cur_val = in_arr[i]; - - if (i == 0) { - cur_val = in_arr[0]; - pos_val = 0; - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&cur_val, sizeof(T)); - tmp_pos_arr[i] = pos_val; - continue; - } - - if (cur_val != in_arr[i - 1]) { - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&cur_val, sizeof(T)); - pos_val += 1; - } - tmp_pos_arr[i] = pos_val; - } - - generated::transpose::fallback::scalar::transpose_i(tmp_pos_arr, pos_arr); - - stt.cur_src_buff += 1; - stt.cur_des_buff += 1; - - FLS_PLOG_KEY_VALUE("pos_arr", ToStr::to_str(pos_arr)); - FLS_PLOG_KEY_VALUE("tmp_pos_arr", ToStr::to_str(tmp_pos_arr)); -} - -template -cmpr_fun_t rle::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return rle_compress; - } - if constexpr (std::is_same()) { - return rle_compress; - } - if constexpr (std::is_same()) { - return rle_compress; - } - if constexpr (std::is_same()) { - return rle_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, rle::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rle/rle_decompress.cpp b/src/cor/prm/rle/rle_decompress.cpp deleted file mode 100644 index 8db7da74..00000000 --- a/src/cor/prm/rle/rle_decompress.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rle_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include - -namespace fastlanes { -template -static void rle_decompress(PageParam src, VecParam des, DecompressState& stt) { - auto* in_pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_pos_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); - auto* in_val_arr = reinterpret_cast(src.arr[stt.cur_des_arr - 1]); - auto* out_val_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 1]); - - // [FIXME] - // zero copy, - // not possible yet! after memory pool. - // out_val_arr = in_val_arr; - // out_pos_arr = in_pos_arr; - - std::memcpy(out_pos_arr, in_pos_arr, vec_bsz()); - /* Lest pos_t determine the size of val_arr. */ - std::memcpy(out_val_arr, in_val_arr, (in_pos_arr[1023] + 1) * sizeof(T)); - - FLS_PLOG_KEY_VALUE("in_val_arr", ToStr::to_str(in_val_arr)); - FLS_PLOG_KEY_VALUE("in_pos_arr", ToStr::to_str(in_pos_arr)); - - FLS_PLOG_KEY_VALUE("out_val_arr", ToStr::to_str(out_val_arr)); - FLS_PLOG_KEY_VALUE("out_pos_arr", ToStr::to_str(out_pos_arr)); - - stt.cur_src_arr -= 2; - stt.cur_des_arr -= 2; -} - -template -de_cmpr_fun_t rle::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return rle_decompress; - } - if constexpr (std::is_same()) { - return rle_decompress; - } - if constexpr (std::is_same()) { - return rle_decompress; - } - if constexpr (std::is_same()) { - return rle_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, rle::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rsum/CMakeLists.txt b/src/cor/prm/rsum/CMakeLists.txt deleted file mode 100644 index b506fec3..00000000 --- a/src/cor/prm/rsum/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_library(fls_rsum_prm - OBJECT - rsum_decompress.cpp - rsum_compress.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_rsum_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_rsum_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/rsum/rsum_compress.cpp b/src/cor/prm/rsum/rsum_compress.cpp deleted file mode 100644 index c6f1746a..00000000 --- a/src/cor/prm/rsum/rsum_compress.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rsum_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/unrsum/unrsum.hpp" - -namespace fastlanes { -template -static void rsum_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 0].mutable_data()); - - /* delta_bases_arr -> des[i + 0] */ - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(in_arr, 1024 / 8); - - /* rsum_data -> src[i + 0] */ - generated::unrsum::fallback::scalar::unrsum_inplace(in_arr); - - stt.cur_des_buff += 1; - - FLS_PLOG_KEY_VALUE("in_arr after unrsum", ToStr::to_str(in_arr)); -} - -template -cmpr_fun_t rsum::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } - - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, rsum::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rsum/rsum_decompress.cpp b/src/cor/prm/rsum/rsum_decompress.cpp deleted file mode 100644 index 16af2f00..00000000 --- a/src/cor/prm/rsum/rsum_decompress.cpp +++ /dev/null @@ -1,51 +0,0 @@ -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rsum_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls_gen/rsum/rsum.hpp" - -namespace fastlanes { -template -static void rsum_decompress(PageParam src, VecParam des, DecompressState& stt) { - auto* in_p = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr - 1]); - auto* delta_bases_arr = reinterpret_cast(src.arr[stt.cur_src_arr]); - - generated::rsum::fallback::scalar::rsum(in_p, out_p, delta_bases_arr); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(reinterpret_cast(out_p))) - FLS_PLOG_KEY_VALUE("delta_bases_arr", ToStr::to_str(delta_bases_arr, 16)); - - stt.cur_src_arr -= 1; - stt.cur_des_arr -= 2; -} - -template -de_cmpr_fun_t rsum::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } - - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, rsum::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/trs/CMakeLists.txt b/src/cor/prm/trs/CMakeLists.txt deleted file mode 100644 index 10070655..00000000 --- a/src/cor/prm/trs/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_trs_prm - OBJECT - trs_decompress.cpp - trs_compress.cpp) - -target_compile_definitions(fls_trs_prm PRIVATE IS_SCALAR) -target_compile_options(fls_trs_prm PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(fls_trs_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_trs_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/trs/trs_compress.cpp b/src/cor/prm/trs/trs_compress.cpp deleted file mode 100644 index decec203..00000000 --- a/src/cor/prm/trs/trs_compress.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/trs_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls_gen/transpose/transpose.hpp" -#include - -namespace fastlanes { -template -static void trs_cmpr(Vec& src_vec, Vec& des_vec, CompressState& stt) { - T* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - FLS_PLOG_KEY_VALUE("before transpose", ToStr::to_str(in_arr)); - /* Fixme: not efficient. implement inplace_transpose */ - T tmp[vec_n_tup()]; - generated::transpose::fallback::scalar::transpose_i(in_arr, tmp); - std::memcpy(in_arr, tmp, vec_bsz()); - FLS_PLOG_KEY_VALUE("after transpose", ToStr::to_str(in_arr)); -} - -template -cmpr_fun_t trs::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same_v) { - return trs_cmpr; - } - - FLS_ABORT("THIS IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, trs::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/trs/trs_decompress.cpp b/src/cor/prm/trs/trs_decompress.cpp deleted file mode 100644 index 23a549ff..00000000 --- a/src/cor/prm/trs/trs_decompress.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/trs_prm.hpp" - -namespace fastlanes { -template -static void trs_decmpr(PageParam src, VecParam des, DecompressState& stt) { - // auto *in_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - // auto *out_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); -} - -template -de_cmpr_fun_t trs::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } - - FLS_ABORT("THIS IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, trs::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/type.cpp b/src/cor/prm/type.cpp deleted file mode 100644 index 344cdac4..00000000 --- a/src/cor/prm/type.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "fls/cor/prm/prm_type.hpp" -#include - -namespace fastlanes { -std::string to_string(prm_t prm_t) { - switch (prm_t) { - case prm_t::INVALID: - break; - case prm_t::ANALYZE: - return {"ANALYZE"}; - case prm_t::PA_ANALYZE: - return {"PA_ANALYZE"}; - case prm_t::MEM_CPY: - return {"MEM_CPY"}; - case prm_t::TRS: - return {"TRS"}; - case prm_t::MEM_TRS: - return {"MEM_TRS"}; - case prm_t::LL_PATCH: - return {"LL_PATCH"}; - case prm_t::B_FFOR: - return {"B_FFOR"}; - case prm_t::RLE: - case prm_t::RSUM: - case prm_t::BITPACK: - case prm_t::FACTOR: - case prm_t::FSST: - case prm_t::B_PATCH: - case prm_t::S_PATCH: - case prm_t::FFOR: - case prm_t::ZER_CPY: - // TODO [FIXME] - default: - break; - } - - return {}; -} -} // namespace fastlanes diff --git a/src/expression/CMakeLists.txt b/src/expression/CMakeLists.txt index b8556931..be234fa3 100644 --- a/src/expression/CMakeLists.txt +++ b/src/expression/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(fls_expression alp_expression.cpp analyze_operator.cpp # CMakeLists.txt + data_parallelize_patch_operator.cpp cross_rle_operator.cpp data_type.cpp decoding_operator.cpp diff --git a/src/expression/alp_expression.cpp b/src/expression/alp_expression.cpp index 5a40d123..40cc2fe2 100644 --- a/src/expression/alp_expression.cpp +++ b/src/expression/alp_expression.cpp @@ -4,6 +4,7 @@ #include "fls/expression/decoding_operator.hpp" #include "fls/expression/physical_expression.hpp" #include "fls/primitive/bitpack/bitpack.hpp" +#include "fls/primitive/patch/patch.hpp" #include "fls/reader/column_view.hpp" #include "fls/reader/segment.hpp" #include "fls/unffor.hpp" @@ -30,6 +31,9 @@ enc_alp_opr::enc_alp_opr(const PhysicalExpr& expr, exp_segment = make_unique(); pos_segment = make_unique(); n_exp_segment = make_unique(); + // + out_count_segment = make_unique(); + out_offset_segment = make_unique(); } template @@ -59,6 +63,11 @@ void enc_alp_opr::Encode() { alp::encoder::analyze_ffor(encoded_arr, alp_state.bit_width, &base); ffor::ffor(encoded_arr, ffor_arr, alp_state.bit_width, &base); + if (data_parallelize) { + Patch::data_parallelize( + exc_arr, pos_arr, alp_state.n_exceptions, out_exc_arr, out_offset, out_pos_arr, out_count); + } + const n_t bytes = calculate_bitpacked_vector_size(alp_state.bit_width); const n_t exc_arr_size = calculate_alp_exception_vector_size(alp_state.n_exceptions); const n_t pos_arr_size = calculate_alp_pos_vector_size(alp_state.n_exceptions); @@ -66,11 +75,28 @@ void enc_alp_opr::Encode() { bw_segment->Flush(&alp_state.bit_width, sizeof(bw_t)); base_segment->Flush(&base, sizeof(typename alp::inner_t::st)); ffor_segment->Flush(ffor_arr, bytes); - exception_segment->Flush(exc_arr, exc_arr_size); + + if (data_parallelize) { + exception_segment->Flush(out_exc_arr, exc_arr_size); + } else { + exception_segment->Flush(exc_arr, exc_arr_size); + } + fac_segment->Flush(&alp_state.fac, sizeof(uint8_t)); exp_segment->Flush(&alp_state.exp, sizeof(uint8_t)); - pos_segment->Flush(pos_arr, pos_arr_size); + + if (data_parallelize) { + pos_segment->Flush(out_pos_arr, pos_arr_size); + } else { + pos_segment->Flush(pos_arr, pos_arr_size); + } + n_exp_segment->Flush(&alp_state.n_exceptions, sizeof(uint16_t)); + + if (data_parallelize) { + out_count_segment->Flush(out_count, 32); + out_offset_segment->Flush(out_offset, 64); + }; } template @@ -84,6 +110,10 @@ void enc_alp_opr::MoveSegments(vector>& segments) { segments.push_back(std::move(exp_segment)); segments.push_back(std::move(pos_segment)); segments.push_back(std::move(n_exp_segment)); + if (data_parallelize) { + segments.push_back(std::move(out_count_segment)); + segments.push_back(std::move(out_offset_segment)); + } } template struct enc_alp_opr; diff --git a/src/expression/data_parallelize_patch_operator.cpp b/src/expression/data_parallelize_patch_operator.cpp new file mode 100644 index 00000000..0c139a63 --- /dev/null +++ b/src/expression/data_parallelize_patch_operator.cpp @@ -0,0 +1,34 @@ +#include "fls/expression/data_parallelize_patch_operator.hpp" +#include "fls/common/string.hpp" +#include "fls/cor/lyt/buf.hpp" +#include "fls/expression/alp_expression.hpp" +#include "fls/expression/decoding_operator.hpp" +#include "fls/expression/interpreter.hpp" +#include "fls/expression/physical_expression.hpp" +#include "fls/primitive/patch/patch.hpp" +#include "fls/reader/column_view.hpp" +#include "fls/reader/segment.hpp" +#include "fls/unffor.hpp" +#include + +namespace fastlanes { +/*--------------------------------------------------------------------------------------------------------------------*\ + * enc_data_parallel_patch_opr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +enc_data_parallel_patch_opr::enc_data_parallel_patch_opr(const PhysicalExpr& expr, + const col_pt& column, + ColumnDescriptorT& column_descriptor, + InterpreterState& state) { + visit(overloaded { + [&](const sp>& opr) { opr->data_parallelize = true; }, + [&](std::monostate&) { FLS_UNREACHABLE(); }, + [&](auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg); }, + }, + expr.operators[state.cur_operator++]); +} + +template struct enc_data_parallel_patch_opr; +template struct enc_data_parallel_patch_opr; + +} // namespace fastlanes diff --git a/src/expression/expression_executor.cpp b/src/expression/expression_executor.cpp index 3b1c771e..13c14dfe 100644 --- a/src/expression/expression_executor.cpp +++ b/src/expression/expression_executor.cpp @@ -1,6 +1,7 @@ #include "fls/expression/expression_executor.hpp" #include "fls/expression/alp_expression.hpp" #include "fls/expression/analyze_operator.hpp" +#include "fls/expression/data_parallelize_patch_operator.hpp" #include "fls/expression/decoding_operator.hpp" #include "fls/expression/encoding_operator.hpp" #include "fls/expression/frequency_operator.hpp" @@ -219,6 +220,9 @@ struct operator_visitor { opr->to_validitymask(); } // + template + void operator()(sp>& opr) { + } void operator()(std::monostate&) { FLS_UNREACHABLE(); } diff --git a/src/expression/interpreter.cpp b/src/expression/interpreter.cpp index a587f9f8..aadb422d 100644 --- a/src/expression/interpreter.cpp +++ b/src/expression/interpreter.cpp @@ -5,6 +5,7 @@ #include "fls/expression/alp_expression.hpp" #include "fls/expression/analyze_operator.hpp" #include "fls/expression/cross_rle_operator.hpp" +#include "fls/expression/data_parallelize_patch_operator.hpp" #include "fls/expression/decoding_operator.hpp" #include "fls/expression/dict_expression.hpp" #include "fls/expression/encoding_operator.hpp" @@ -348,6 +349,22 @@ void make_enc_alp_expr(PhysicalExpr& physical_expr, operators.emplace_back(make_shared>(physical_expr, column, column_descriptor, state)); } +/*--------------------------------------------------------------------------------------------------------------------*\ + * make_galp_expr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +void make_enc_galp_expr(PhysicalExpr& physical_expr, + const rowgroup_pt& rowgroup, + ColumnDescriptorT& column_descriptor, + InterpreterState& state) { + auto& operators = physical_expr.operators; + const auto& column = rowgroup[column_descriptor.idx]; + + operators.emplace_back(make_shared>(physical_expr, column, column_descriptor, state)); + operators.emplace_back( + make_shared>(physical_expr, column, column_descriptor, state)); +} + /*--------------------------------------------------------------------------------------------------------------------*\ * make_rle_expr \*--------------------------------------------------------------------------------------------------------------------*/ @@ -613,6 +630,14 @@ sp Interpreter::Encoding::Interpret(ColumnDescriptorT& column_desc make_enc_alp_expr(*physical_expr, physical_rowgroup, column_descriptor, state); break; } + case EXP_GALP_FLT: { + make_enc_galp_expr(*physical_expr, physical_rowgroup, column_descriptor, state); + break; + } + case EXP_GALP_DBL: { + make_enc_galp_expr(*physical_expr, physical_rowgroup, column_descriptor, state); + break; + } case EXP_ALP_RD_DBL: { make_enc_alp_rd_expr(*physical_expr, physical_rowgroup, column_descriptor, state); break; @@ -1125,6 +1150,15 @@ void make_dec_alp_expr(PhysicalExpr& physical_expr, const ColumnView& column_vie physical_expr.operators.emplace_back(make_shared>(column_view, state)); } +/*--------------------------------------------------------------------------------------------------------------------*\ + * make_dec_galp_expr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +void make_dec_galp_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { + state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + physical_expr.operators.emplace_back(make_shared>(column_view, state)); +} + /*--------------------------------------------------------------------------------------------------------------------*\ * make_dec_alp_rd_expr \*--------------------------------------------------------------------------------------------------------------------*/ @@ -1616,6 +1650,14 @@ void Interpreter::Decoding::Interpret(const ColumnDescriptorT& column_descriptor make_dec_alp_expr(physical_expr, column_view, state); break; } + case EXP_GALP_FLT: { + make_dec_galp_expr(physical_expr, column_view, state); + break; + } + case EXP_GALP_DBL: { + make_dec_galp_expr(physical_expr, column_view, state); + break; + } case EXP_ALP_RD_DBL: { make_dec_alp_rd_expr(physical_expr, column_view, state); break; diff --git a/src/expression/physical_expression.cpp b/src/expression/physical_expression.cpp index b4601b1e..e4f3ab5b 100644 --- a/src/expression/physical_expression.cpp +++ b/src/expression/physical_expression.cpp @@ -3,6 +3,7 @@ #include "fls/expression/alp_expression.hpp" #include "fls/expression/analyze_operator.hpp" #include "fls/expression/cross_rle_operator.hpp" +#include "fls/expression/data_parallelize_patch_operator.hpp" #include "fls/expression/decoding_operator.hpp" #include "fls/expression/dict_expression.hpp" #include "fls/expression/encoding_operator.hpp" @@ -218,6 +219,10 @@ struct point_to_visitor { template void operator()(const sp>& opr) { } + // + template + void operator()(const sp>& opr) { + } void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } @@ -360,7 +365,10 @@ struct flush_segments_visitor { void operator()(const sp& opr) { opr->MoveSegments(segments); } - + // + template + void operator()(const sp>& opr) { + } void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } @@ -514,6 +522,10 @@ struct extract_segments_visitor { opr->MoveSegments(segments); } // + template + void operator()(const sp>& opr) { + } + // void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } @@ -651,6 +663,10 @@ struct finalize_operators_visitor { void operator()(const sp& opr) { } // + template + void operator()(const sp>& opr) { + } + // void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } diff --git a/src/expression/rpn.cpp b/src/expression/rpn.cpp index e53fe994..8deca12a 100644 --- a/src/expression/rpn.cpp +++ b/src/expression/rpn.cpp @@ -318,6 +318,12 @@ string token_to_string(OperatorToken token) { return "EXP_RLE_U08_SLPATCH_U16"; // 152 case OperatorToken::EXP_DELTA_U08: return "EXP_DELTA_U08"; // 153 + case OperatorToken::EXP_RLE_U08_U16: + return "EXP_RLE_U08_U16"; // 154 + case OperatorToken::EXP_GALP_FLT: + return "EXP_GALP_FLT"; // 155 + case OperatorToken::EXP_GALP_DBL: + return "EXP_GALP_DBL"; // 156 default: FLS_UNREACHABLE() } diff --git a/src/include/fls/cuda/common.hpp b/src/include/fls/cuda/common.hpp new file mode 100644 index 00000000..c8ba4d33 --- /dev/null +++ b/src/include/fls/cuda/common.hpp @@ -0,0 +1,32 @@ +#ifndef FLS_CUDA_COMMON_HPP +#define FLS_CUDA_COMMON_HPP + +#include "fls/cfg/cfg.hpp" +#include "fls/common/alias.hpp" +#include "fls/cuda/config.hpp" + +namespace fastlanes::cuda { + +template +constexpr n_t bits_in_object() { + return sizeof(PT) * CHAR_BIT; +} + +template +constexpr n_t get_lane_bitwidth() { + return bits_in_object(); +} + +template +constexpr n_t get_n_lanes() { + return Config::REGISTER_WIDTH / get_lane_bitwidth(); +} + +template +constexpr n_t get_values_per_lane() { + return CFG::VEC_SZ / get_n_lanes(); +} + +} // namespace fastlanes::cuda + +#endif // FLS_CUDA_COMMON_HPP \ No newline at end of file diff --git a/src/include/fls/cuda/config.hpp b/src/include/fls/cuda/config.hpp new file mode 100644 index 00000000..9342caa7 --- /dev/null +++ b/src/include/fls/cuda/config.hpp @@ -0,0 +1,16 @@ +#ifndef FLS_CUDA_CONFIG_HPP +#define FLS_CUDA_CONFIG_HPP + +#include "fls/common/alias.hpp" // for hdr_field_t + +namespace fastlanes::cuda { +class Config { +public: + explicit Config(); // +public: + static constexpr n_t REGISTER_WIDTH = 1024; // +}; + +} // namespace fastlanes::cuda + +#endif // FLS_CUDA_CONFIG_HPP diff --git a/src/include/fls/expression/alp_expression.hpp b/src/include/fls/expression/alp_expression.hpp index 4215fae9..8c9a3326 100644 --- a/src/include/fls/expression/alp_expression.hpp +++ b/src/include/fls/expression/alp_expression.hpp @@ -3,6 +3,7 @@ #include "alp.hpp" #include "fls/cfg/cfg.hpp" +#include "fls/cuda/common.hpp" #include "fls/reader/segment.hpp" #include "fls/table/chunk.hpp" #include "fls/table/rowgroup.hpp" @@ -42,6 +43,10 @@ struct enc_alp_opr { alignas(64) uint16_t pos_arr[CFG::VEC_SZ]; alignas(64) PT exc_arr[CFG::VEC_SZ]; + // + bool data_parallelize {false}; + + // cuda up ffor_segment; up base_segment; up bw_segment; @@ -50,6 +55,16 @@ struct enc_alp_opr { up exp_segment; up pos_segment; up n_exp_segment; + + // + // arrays + alignas(64) uint8_t out_count[cuda::get_n_lanes()]; + alignas(64) uint16_t out_offset[cuda::get_n_lanes()]; + alignas(64) uint16_t out_pos_arr[CFG::VEC_SZ]; + alignas(64) PT out_exc_arr[CFG::VEC_SZ]; + + up out_count_segment; + up out_offset_segment; }; /*--------------------------------------------------------------------------------------------------------------------*\ diff --git a/src/include/fls/expression/data_parallelize_patch_operator.hpp b/src/include/fls/expression/data_parallelize_patch_operator.hpp new file mode 100644 index 00000000..18de8340 --- /dev/null +++ b/src/include/fls/expression/data_parallelize_patch_operator.hpp @@ -0,0 +1,32 @@ +#ifndef FLS_EXPRESSION_DATA_PARALLEL_PATCH_OPERATOR_HPP +#define FLS_EXPRESSION_DATA_PARALLEL_PATCH_OPERATOR_HPP + +#include "fls/cuda/common.hpp" +#include "fls/reader/segment.hpp" +#include "fls/table/chunk.hpp" +#include "fls/table/rowgroup.hpp" + +namespace fastlanes { +/*--------------------------------------------------------------------------------------------------------------------*/ +class Segment; +struct ColumnDescriptorT; +class PhysicalExpr; +struct InterpreterState; +class ColumnView; +struct InterpreterState; +/*--------------------------------------------------------------------------------------------------------------------*/ + +/*--------------------------------------------------------------------------------------------------------------------*\ + * enc_alp_opr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +struct enc_data_parallel_patch_opr { + explicit enc_data_parallel_patch_opr(const PhysicalExpr& expr, + const col_pt& column, + ColumnDescriptorT& column_descriptor, + InterpreterState& state); +}; + +} // namespace fastlanes + +#endif // FLS_EXPRESSION_DATA_PARALLEL_PATCH_OPERATOR_HPP diff --git a/src/include/fls/expression/physical_expression.hpp b/src/include/fls/expression/physical_expression.hpp index c2275f46..afbc805a 100644 --- a/src/include/fls/expression/physical_expression.hpp +++ b/src/include/fls/expression/physical_expression.hpp @@ -89,6 +89,8 @@ template struct dec_cross_rle_opr; struct enc_validitymask_opr; struct dec_validitymask_opr; +template +struct enc_data_parallel_patch_opr; /*--------------------------------------------------------------------------------------------------------------------*/ using physical_operator = variant, sp>, sp>, - sp> + sp>, + // + // DATA Parallelize Patch + sp>, + sp> // >; diff --git a/src/include/fls/footer/operator_token_generated.h b/src/include/fls/footer/operator_token_generated.h index edb96fe1..7ac5c01b 100644 --- a/src/include/fls/footer/operator_token_generated.h +++ b/src/include/fls/footer/operator_token_generated.h @@ -167,10 +167,12 @@ enum class OperatorToken : uint16_t { EXP_FFOR_SLPATCH_U08 = 151, EXP_RLE_U08_SLPATCH_U16 = 152, EXP_DELTA_U08 = 153, - EXP_RLE_U08_U16 = 154 + EXP_RLE_U08_U16 = 154, + EXP_GALP_FLT = 155, + EXP_GALP_DBL = 156 }; -inline const OperatorToken (&EnumValuesOperatorToken())[155] { +inline const OperatorToken (&EnumValuesOperatorToken())[157] { static const OperatorToken values[] = {OperatorToken::INVALID, OperatorToken::EXP_EQUAL, OperatorToken::EXP_CONSTANT_I64, @@ -325,12 +327,14 @@ inline const OperatorToken (&EnumValuesOperatorToken())[155] { OperatorToken::EXP_FFOR_SLPATCH_U08, OperatorToken::EXP_RLE_U08_SLPATCH_U16, OperatorToken::EXP_DELTA_U08, - OperatorToken::EXP_RLE_U08_U16}; + OperatorToken::EXP_RLE_U08_U16, + OperatorToken::EXP_GALP_FLT, + OperatorToken::EXP_GALP_DBL}; return values; } inline const char* const* EnumNamesOperatorToken() { - static const char* const names[156] = {"INVALID", + static const char* const names[158] = {"INVALID", "EXP_EQUAL", "EXP_CONSTANT_I64", "EXP_CONSTANT_I32", @@ -485,12 +489,14 @@ inline const char* const* EnumNamesOperatorToken() { "EXP_RLE_U08_SLPATCH_U16", "EXP_DELTA_U08", "EXP_RLE_U08_U16", + "EXP_GALP_FLT", + "EXP_GALP_DBL", nullptr}; return names; } inline const char* EnumNameOperatorToken(OperatorToken e) { - if (::flatbuffers::IsOutRange(e, OperatorToken::INVALID, OperatorToken::EXP_RLE_U08_U16)) + if (::flatbuffers::IsOutRange(e, OperatorToken::INVALID, OperatorToken::EXP_GALP_DBL)) return ""; const size_t index = static_cast(e); return EnumNamesOperatorToken()[index]; diff --git a/src/include/fls/primitive/patch/patch.hpp b/src/include/fls/primitive/patch/patch.hpp new file mode 100644 index 00000000..e8080db0 --- /dev/null +++ b/src/include/fls/primitive/patch/patch.hpp @@ -0,0 +1,22 @@ +#ifndef FLS_PRIMITIVE_PATCH_PATCH_HPP +#define FLS_PRIMITIVE_PATCH_PATCH_HPP + +#include "fls/common/common.hpp" + +namespace fastlanes { +n_t calculate_bitpacked_vector_size(bw_t bw); + +template +class Patch { +public: + static void data_parallelize(const PT* in_exc_arr, + const uint16_t* in_pos_arr, + n_t n_exceptions, + PT* out_exc_arr, + uint16_t* out_offset, + uint16_t* out_pos_arr, + uint8_t* out_count); +}; + +} // namespace fastlanes +#endif // FLS_PRIMITIVE_PATCH_PATCH_HPP diff --git a/src/primitive/CMakeLists.txt b/src/primitive/CMakeLists.txt index ba93b886..8ebd50cc 100644 --- a/src/primitive/CMakeLists.txt +++ b/src/primitive/CMakeLists.txt @@ -3,6 +3,7 @@ add_subdirectory(copy) add_subdirectory(fls_memset) add_subdirectory(fsst) add_subdirectory(fsst12) +add_subdirectory(patch) add_subdirectory(predicate) add_subdirectory(rle) add_subdirectory(untraspose) diff --git a/src/primitive/patch/CMakeLists.txt b/src/primitive/patch/CMakeLists.txt new file mode 100644 index 00000000..8c549b8e --- /dev/null +++ b/src/primitive/patch/CMakeLists.txt @@ -0,0 +1,20 @@ +add_library(fls_primitive_patch + OBJECT + patch.cpp +) + + +set(FASTLANES_OBJECT_FILES + ${FASTLANES_OBJECT_FILES} $ + PARENT_SCOPE) + +if (ENABLE_IWYU) + set_property(TARGET fls_primitive_patch PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) +endif () + +target_compile_options(fls_primitive_patch PRIVATE -Wno-macro-redefined) + +target_link_libraries(fls_primitive_patch + PUBLIC + FastLanes::headers +) diff --git a/src/primitive/patch/patch.cpp b/src/primitive/patch/patch.cpp new file mode 100644 index 00000000..5d2b0f21 --- /dev/null +++ b/src/primitive/patch/patch.cpp @@ -0,0 +1,46 @@ +#include "fls/primitive/patch/patch.hpp" +#include "fls/cuda/common.hpp" + +namespace fastlanes { + +template +void Patch::data_parallelize(const PT* __restrict in_exc_arr, + const uint16_t* __restrict in_pos_arr, + n_t n_exceptions, + PT* __restrict out_exc_arr, + uint16_t* __restrict out_offset, + uint16_t* __restrict out_pos_arr, + uint8_t* __restrict out_count) { + // + constexpr n_t N_LANES = cuda::get_n_lanes(); + static_assert((N_LANES & (N_LANES - 1)) == 0, "N_LANES must be a power of two"); + constexpr n_t MASK = N_LANES - 1; + + // + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_count[lane] = 0; + } + + for (n_t i = 0; i < n_exceptions; ++i) { + ++out_count[in_pos_arr[i] & MASK]; + } + + // + uint16_t running = 0; + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_offset[lane] = running; + running += out_count[lane]; + } + + for (n_t i = 0; i < n_exceptions; ++i) { + const n_t lane = in_pos_arr[i] & MASK; + const uint16_t dst = out_offset[lane]++; // advance only the cursor + out_exc_arr[dst] = in_exc_arr[i]; + out_pos_arr[dst] = in_pos_arr[i]; + } +} + +template class Patch; +template class Patch; + +} // namespace fastlanes \ No newline at end of file diff --git a/test/src/expression_tests/CMakeLists.txt b/test/src/expression_tests/CMakeLists.txt index ca9ec939..fb978304 100644 --- a/test/src/expression_tests/CMakeLists.txt +++ b/test/src/expression_tests/CMakeLists.txt @@ -1,30 +1,15 @@ -add_executable(test_frequency frequency_test.cpp) -target_link_libraries(test_frequency PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_frequency COMMAND $) - -add_executable(test_rle rle_test.cpp) -target_link_libraries(test_rle PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_rle COMMAND $) - -add_executable(test_cross_rle cross_rle_test.cpp) -target_link_libraries(test_cross_rle PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_cross_rle COMMAND $) - -add_executable(test_alp alp_test.cpp) -target_link_libraries(test_alp PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_alp COMMAND $) - -add_executable(test_fsst_expr fsst_test.cpp) -target_link_libraries(test_fsst_expr PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_fsst_expr COMMAND $) - -add_executable(test_uncompressed uncompressed_test.cpp) -target_link_libraries(test_uncompressed PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_uncompressed COMMAND $) - -fls_enable_sanitizers(test_frequency) -fls_enable_sanitizers(test_rle) -fls_enable_sanitizers(test_cross_rle) -fls_enable_sanitizers(test_alp) -fls_enable_sanitizers(test_fsst_expr) -fls_enable_sanitizers(test_uncompressed) +macro(fls_add_test NAME SRC) + add_executable(test_${NAME} ${SRC}) + target_link_libraries(test_${NAME} PUBLIC gtest_main gmock_main FastLanes) + add_test(NAME test_${NAME} COMMAND $) + fls_enable_sanitizers(test_${NAME}) +endmacro() + +fls_add_test(alp alp_test.cpp) +# +fls_add_test(cross_rle cross_rle_test.cpp) +fls_add_test(frequency frequency_test.cpp) +fls_add_test(fsst_expr fsst_test.cpp) +fls_add_test(galp galp_test.cpp) +fls_add_test(rle rle_test.cpp) +fls_add_test(uncompressed uncompressed_test.cpp) diff --git a/test/src/expression_tests/galp_test.cpp b/test/src/expression_tests/galp_test.cpp new file mode 100644 index 00000000..f113712c --- /dev/null +++ b/test/src/expression_tests/galp_test.cpp @@ -0,0 +1,57 @@ +#include "fls_tester.hpp" + +namespace fastlanes { + +// ——— FLOAT tests ——————————————————————————————————————————————— + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_ONE_VECTOR) { + TestCorrectness(galp::float_galp_one_vector, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_SINGLE_COLUMN) { + TestCorrectness(galp::float_galp_single_column, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_CONSTANT_ONE_VECTOR) { + TestCorrectness(galp::float_constant_one_vector, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_CONSTANT_SINGLE_COLUMN) { + TestCorrectness(galp::float_constant_single_column, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_POSITIVE_INF_ONE_VECTOR) { + TestCorrectness(galp::float_positive_inf_one_vector, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_POSITIVE_INF_SINGLE_COLUMN) { + TestCorrectness(galp::float_positive_inf_single_column, {OperatorToken::EXP_GALP_FLT}); +} + +// ——— DOUBLE tests ——————————————————————————————————————————————— + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_ONE_VECTOR) { + TestCorrectness(galp::double_galp_one_vector, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_SINGLE_COLUMN) { + TestCorrectness(galp::double_galp_single_column, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_CONSTANT_ONE_VECTOR) { + TestCorrectness(galp::double_constant_one_vector, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_CONSTANT_SINGLE_COLUMN) { + TestCorrectness(galp::double_constant_single_column, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_POSITIVE_INF_ONE_VECTOR) { + TestCorrectness(galp::double_positive_inf_one_vector, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_POSITIVE_INF_SINGLE_COLUMN) { + TestCorrectness(galp::double_positive_inf_single_column, {OperatorToken::EXP_GALP_DBL}); +} + +} // namespace fastlanes diff --git a/test/src/primitive_tests/CMakeLists.txt b/test/src/primitive_tests/CMakeLists.txt index 6c12846d..336535d2 100644 --- a/test/src/primitive_tests/CMakeLists.txt +++ b/test/src/primitive_tests/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable(primitive_test fsst_test.cpp + patch_test.cpp ) target_link_libraries(primitive_test diff --git a/test/src/primitive_tests/patch_test.cpp b/test/src/primitive_tests/patch_test.cpp new file mode 100644 index 00000000..8bdc965b --- /dev/null +++ b/test/src/primitive_tests/patch_test.cpp @@ -0,0 +1,163 @@ +// test/src/primitive_tests/patch_test.cpp +// Google Test suite for fastlanes::Patch::data_parallelize +// Updated for API that returns **per‑lane counts (uint8_t) and offsets (uint16_t)**. +// Scenarios: zero exceptions, single exception, one‑per‑lane, 1024 sequential. + +#include "fls/cuda/common.hpp" +#include "fls/primitive/patch/patch.hpp" +#include +#include + +namespace fastlanes { + +// ----------------------------------------------------------------------------- +// Typed‑test fixture +// ----------------------------------------------------------------------------- + +template +class PatchDataParallelizeTest : public ::testing::Test { +protected: + Patch patch; +}; + +using TestTypes = ::testing::Types; +TYPED_TEST_SUITE(PatchDataParallelizeTest, TestTypes); + +// ----------------------------------------------------------------------------- +// 1. Zero exceptions ---------------------------------------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, HandlesZeroExceptions) { + using PT = TypeParam; + + constexpr n_t n_exceptions = 0; + std::vector dummy_exc(1); + std::vector dummy_pos(1); + + std::vector out_exc(1); + std::vector out_pos(1); + const n_t N_LANES = cuda::get_n_lanes(); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize(dummy_exc.data(), + dummy_pos.data(), + n_exceptions, + out_exc.data(), + lane_end.data(), + out_pos.data(), + lane_cnt.data()); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + ASSERT_EQ(lane_cnt[lane], 0); + ASSERT_EQ(lane_end[lane], static_cast(0)); // inclusive prefix + } +} + +// ----------------------------------------------------------------------------- +// 2. Single exception routed correctly ---------------------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, SingleExceptionRoutedToCorrectLane) { + using PT = TypeParam; + + const std::vector in_exc = {42}; + const std::vector in_pos = {3}; + constexpr n_t n_exceptions = 1; + + std::vector out_exc(n_exceptions); + std::vector out_pos(n_exceptions); + const n_t N_LANES = cuda::get_n_lanes(); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize( + in_exc.data(), in_pos.data(), n_exceptions, out_exc.data(), lane_end.data(), out_pos.data(), lane_cnt.data()); + + ASSERT_EQ(out_exc[0], in_exc[0]); + ASSERT_EQ(out_pos[0], in_pos[0]); + + const n_t exc_lane = in_pos[0] & (N_LANES - 1); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + // counts + ASSERT_EQ(lane_cnt[lane], static_cast(lane == exc_lane)); + + // inclusive offset: 0 for lanes < exc_lane, 1 otherwise + const uint16_t expected_end = (lane < exc_lane) ? 0 : 1; + ASSERT_EQ(lane_end[lane], expected_end); + } +} + +// ----------------------------------------------------------------------------- +// 3. One exception per lane --------------------------------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, DistributesAcrossAllLanes) { + using PT = TypeParam; + const n_t N_LANES = cuda::get_n_lanes(); + + std::vector in_exc; + std::vector in_pos; + in_exc.reserve(N_LANES); + in_pos.reserve(N_LANES); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + in_exc.push_back(static_cast(lane + 10)); + in_pos.push_back(static_cast(lane)); + } + + const n_t n_exceptions = N_LANES; + std::vector out_exc(n_exceptions); + std::vector out_pos(n_exceptions); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize( + in_exc.data(), in_pos.data(), n_exceptions, out_exc.data(), lane_end.data(), out_pos.data(), lane_cnt.data()); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + ASSERT_EQ(lane_cnt[lane], 1); + ASSERT_EQ(lane_end[lane], static_cast(lane + 1)); // inclusive + } + + for (n_t i = 0; i < n_exceptions; ++i) { + ASSERT_EQ(out_exc[i], in_exc[i]); + ASSERT_EQ(out_pos[i], in_pos[i]); + } +} + +// ----------------------------------------------------------------------------- +// 4. 1024 sequential exceptions (offset‑wrap stress) -------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, HandlesThousandTwentyFourSequential) { + using PT = TypeParam; + constexpr n_t n_exceptions = 1024; + + std::vector in_exc(n_exceptions); + std::vector in_pos(n_exceptions); + for (n_t i = 0; i < n_exceptions; ++i) { + in_exc[i] = static_cast(i); + in_pos[i] = static_cast(i); + } + + std::vector out_exc(n_exceptions); + std::vector out_pos(n_exceptions); + const n_t N_LANES = cuda::get_n_lanes(); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize( + in_exc.data(), in_pos.data(), n_exceptions, out_exc.data(), lane_end.data(), out_pos.data(), lane_cnt.data()); + + n_t begin = 0; + for (n_t lane = 0; lane < N_LANES; ++lane) { + const n_t expected_end = begin + lane_cnt[lane]; + ASSERT_EQ(lane_end[lane], static_cast(expected_end)); + begin = expected_end; + } + ASSERT_EQ(begin, n_exceptions); // final end == total exceptions +} + +} // namespace fastlanes diff --git a/test/src/quick_fuzz_tests/fuzz_config.json b/test/src/quick_fuzz_tests/fuzz_config.json index 0dc30769..80f49e46 100644 --- a/test/src/quick_fuzz_tests/fuzz_config.json +++ b/test/src/quick_fuzz_tests/fuzz_config.json @@ -1,6 +1,6 @@ { "num_cases": 10, - "base_seed": 2, + "base_seed": 3, "delimiter": "|", "min_cols": 1, "max_cols": 2,