From f25ad9f58338fdeb7bc44d4eaa10becadc6aaae2 Mon Sep 17 00:00:00 2001 From: azimafroozeh Date: Tue, 22 Jul 2025 21:56:37 +0200 Subject: [PATCH 1/3] add GALP --- .clang-format | 12 + .github/CODEOWNERS | 1 + .github/workflows/cuda.yml | 58 + .github/workflows/galp.yaml | 114 + CMakeLists.txt | 36 +- README.md | 26 +- benchmark/CMakeLists.txt | 2 + benchmark/bench_galp/CMakeLists.txt | 2 + .../bench_data_parallelize_patch.cpp | 230 + .../result/galp/benchmark_new_vs_old.png | Bin 0 -> 77077 bytes benchmark/result/galp/patch.csv | 127 + benchmark/result/galp/patch.metadata | 26 + .../result/galp/patch/README.md | 0 benchmark/result/galp/plot.py | 154 + cuda/CMakeLists.txt | 44 + cuda/src/cuda_reader.cu | 19 + data/include/data/fastlanes_data.hpp | 1 + data/include/data/galp.hpp | 64 + flatbuffers_schemas/operator_token.fbs | 2 + galp/CMakeLists.txt | 92 + galp/README.md | 3 + galp/benchmark/CMakeLists.txt | 46 + galp/benchmark/compressors-benchmarks.cu | 248 + .../alp-double-decompress_column-bindings.cu | 105 + .../alp-double-query_column-bindings.cu | 106 + .../alp-double-query_multi_column-bindings.cu | 50 + .../alp-float-decompress_column-bindings.cu | 146 + .../alp-float-query_column-bindings.cu | 147 + .../alp-float-query_multi_column-bindings.cu | 65 + ...ended-double-decompress_column-bindings.cu | 134 + ...lpextended-double-query_column-bindings.cu | 135 + ...nded-double-query_multi_column-bindings.cu | 135 + ...tended-float-decompress_column-bindings.cu | 189 + ...alpextended-float-query_column-bindings.cu | 190 + ...ended-float-query_multi_column-bindings.cu | 190 + .../bp-uint32_t-decompress_column-bindings.cu | 85 + .../bp-uint32_t-query_column-bindings.cu | 86 + .../bp-uint64_t-decompress_column-bindings.cu | 74 + .../bp-uint64_t-query_column-bindings.cu | 74 + .../ffor-uint32_t-compute_column-bindings.cu | 91 + ...for-uint32_t-decompress_column-bindings.cu | 86 + .../ffor-uint32_t-query_column-bindings.cu | 86 + ...or-uint32_t-query_multi_column-bindings.cu | 60 + .../ffor-uint64_t-compute_column-bindings.cu | 78 + ...for-uint64_t-decompress_column-bindings.cu | 74 + .../ffor-uint64_t-query_column-bindings.cu | 73 + ...or-uint64_t-query_multi_column-bindings.cu | 48 + .../generated-bindings/kernel-bindings.cuh | 211 + .../multi-column-device-kernels.cuh | 527 + galp/benchmark/micro-benchmarks.cu | 261 + .../benchmark/nvcomp/benchmark-compressors.cu | 211 + galp/benchmark/nvcomp/nvcomp-compressors.cu | 80 + .../generate-kernel-bindings.py | 429 + .../generate-multicolumn-kernels.py | 109 + galp/src/CMakeLists.txt | 12 + galp/src/alp/alp-bindings.cu | 263 + galp/src/engine/enums.cu | 163 + galp/src/include/alp/alp-bindings.cuh | 37 + galp/src/include/engine/data.cuh | 545 + galp/src/include/engine/device-utils.cuh | 140 + galp/src/include/engine/enums.cuh | 94 + galp/src/include/engine/kernels.cuh | 152 + .../engine/multi-column-host-kernels.cuh | 141 + galp/src/include/engine/verification.cuh | 118 + galp/src/include/flsgpu/alp.cuh | 461 + galp/src/include/flsgpu/consts.cuh | 33 + galp/src/include/flsgpu/device-types.cuh | 11 + galp/src/include/flsgpu/fls-switch-case.cuh | 891 + galp/src/include/flsgpu/fls.cuh | 648 + galp/src/include/flsgpu/flsgpu-api.cuh | 4 + galp/src/include/flsgpu/host-utils.cuh | 113 + galp/src/include/flsgpu/old-fls.cuh | 6936 +++++ galp/src/include/flsgpu/structs.cuh | 411 + galp/src/include/flsgpu/utils.cuh | 104 + .../include/generator/generate_binaries.hpp | 116 + .../include/nvcomp/benchmark-compressors.cuh | 141 + .../src/include/nvcomp/nvcomp-compressors.cuh | 125 + galp/test/CMakeLists.txt | 18 + galp/test/galp_test.cu | 232 + galp/test/thrust_core_test.cu | 118 + galp/test/thrust_memcpy_test.cu | 25 + gpu/CMakeLists.txt | 1 - gpu/cuda/CMakeLists.txt | 1 - mk/benchmark.mk | 34 + mk/format.mk | 4 +- scripts/generate_synthetic_data.py | 3 +- scripts/generator_helpers/galp_generator.py | 76 + src/connection.cpp | 15 +- src/cor/compression_plan_history.cpp | 16 - src/cor/cpu.cpp | 3 - src/cor/lyt/col.cpp | 62 - src/cor/lyt/dic/CMakeLists.txt | 13 - src/cor/lyt/dic/dic.cpp | 275 - src/cor/lyt/dic/dic_fac.cpp | 581 - src/cor/lyt/page/CMakeLists.txt | 17 - src/cor/lyt/page/entry_point.cpp | 65 - src/cor/lyt/page/offset.cpp | 32 - src/cor/lyt/page/page.cpp | 181 - src/cor/lyt/page/page_hdr.cpp | 29 - src/cor/lyt/page/page_param.cpp | 13 - src/cor/lyt/page/type.cpp | 19 - src/cor/lyt/vec.cpp | 365 - src/cor/mtd.cpp | 85 - src/cor/prm/CMakeLists.txt | 3 +- src/cor/prm/alp/CMakeLists.txt | 18 - src/cor/prm/alp/alp_compress.cpp | 58 - src/cor/prm/alp/alp_decompress.cpp | 54 - src/cor/prm/b_ffor/CMakeLists.txt | 18 - src/cor/prm/b_ffor/b_ffor_compress.cpp | 38 - src/cor/prm/b_ffor/b_ffor_decompress.cpp | 44 - src/cor/prm/bitpack/CMakeLists.txt | 20 - src/cor/prm/bitpack/bitpack_compress.cpp | 22254 ---------------- src/cor/prm/bitpack/bitpack_decompress.cpp | 17303 ------------ src/cor/prm/factor/CMakeLists.txt | 15 - src/cor/prm/factor/compress.cpp | 21 - src/cor/prm/factor/decompress.cpp | 6 - src/cor/prm/ffor/CMakeLists.txt | 14 - src/cor/prm/ffor/ffor_analyze.cpp | 202 - src/cor/prm/ffor/ffor_compress.cpp | 60 - src/cor/prm/ffor/ffor_decompress.cpp | 58 - src/cor/prm/mem_cpy/CMakeLists.txt | 15 - src/cor/prm/mem_cpy/mem_cpy_compress.cpp | 64 - src/cor/prm/mem_cpy/mem_cpy_decompress.cpp | 66 - src/cor/prm/mem_trs/CMakeLists.txt | 15 - src/cor/prm/mem_trs/mem_trs_compress.cpp | 53 - src/cor/prm/mem_trs/mem_trs_decompress.cpp | 49 - src/cor/prm/off_cpy/CMakeLists.txt | 1 - src/cor/prm/off_cpy/off_cpy_compress.cpp | 0 src/cor/prm/off_cpy/off_cpy_decompress.cpp | 0 src/cor/prm/patch/CMakeLists.txt | 18 - src/cor/prm/patch/b_patch/CMakeLists.txt | 15 - .../prm/patch/b_patch/b_patch_compress.cpp | 49 - .../prm/patch/b_patch/b_patch_decompress.cpp | 54 - src/cor/prm/patch/ll_patch/CMakeLists.txt | 16 - .../prm/patch/ll_patch/ll_patch_compress.cpp | 83 - .../patch/ll_patch/ll_patch_decompress.cpp | 67 - src/cor/prm/patch/patch.cpp | 152 - src/cor/prm/patch/s_patch/CMakeLists.txt | 15 - .../prm/patch/s_patch/s_patch_compress.cpp | 47 - .../prm/patch/s_patch/s_patch_decompress.cpp | 55 - src/cor/prm/rle/CMakeLists.txt | 13 - src/cor/prm/rle/rle_compress.cpp | 67 - src/cor/prm/rle/rle_decompress.cpp | 63 - src/cor/prm/rsum/CMakeLists.txt | 13 - src/cor/prm/rsum/rsum_compress.cpp | 50 - src/cor/prm/rsum/rsum_decompress.cpp | 51 - src/cor/prm/trs/CMakeLists.txt | 15 - src/cor/prm/trs/trs_compress.cpp | 53 - src/cor/prm/trs/trs_decompress.cpp | 44 - src/cor/prm/type.cpp | 39 - src/expression/CMakeLists.txt | 1 + src/expression/alp_expression.cpp | 34 +- .../data_parallelize_patch_operator.cpp | 34 + src/expression/expression_executor.cpp | 4 + src/expression/interpreter.cpp | 42 + src/expression/physical_expression.cpp | 18 +- src/expression/rpn.cpp | 6 + src/include/fls/cuda/common.hpp | 32 + src/include/fls/cuda/config.hpp | 16 + src/include/fls/expression/alp_expression.hpp | 15 + .../data_parallelize_patch_operator.hpp | 32 + .../fls/expression/physical_expression.hpp | 8 +- .../fls/footer/operator_token_generated.h | 16 +- src/include/fls/primitive/patch/patch.hpp | 22 + src/primitive/CMakeLists.txt | 1 + src/primitive/patch/CMakeLists.txt | 20 + src/primitive/patch/patch.cpp | 46 + test/src/expression_tests/CMakeLists.txt | 45 +- test/src/expression_tests/galp_test.cpp | 57 + test/src/primitive_tests/CMakeLists.txt | 1 + test/src/primitive_tests/patch_test.cpp | 163 + 171 files changed, 18377 insertions(+), 43248 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/cuda.yml create mode 100644 .github/workflows/galp.yaml create mode 100644 benchmark/bench_galp/CMakeLists.txt create mode 100644 benchmark/bench_galp/bench_data_parallelize_patch.cpp create mode 100644 benchmark/result/galp/benchmark_new_vs_old.png create mode 100644 benchmark/result/galp/patch.csv create mode 100644 benchmark/result/galp/patch.metadata rename src/cor/exp_pool.cpp => benchmark/result/galp/patch/README.md (100%) create mode 100644 benchmark/result/galp/plot.py create mode 100644 cuda/CMakeLists.txt create mode 100644 cuda/src/cuda_reader.cu create mode 100644 data/include/data/galp.hpp create mode 100644 galp/CMakeLists.txt create mode 100644 galp/README.md create mode 100644 galp/benchmark/CMakeLists.txt create mode 100644 galp/benchmark/compressors-benchmarks.cu create mode 100644 galp/benchmark/generated-bindings/alp-double-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alp-double-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alp-double-query_multi_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alp-float-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alp-float-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alp-float-query_multi_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alpextended-double-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alpextended-double-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alpextended-double-query_multi_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alpextended-float-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alpextended-float-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/alpextended-float-query_multi_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/bp-uint32_t-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/bp-uint32_t-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/bp-uint64_t-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/bp-uint64_t-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint32_t-compute_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint32_t-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint32_t-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint64_t-compute_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint64_t-decompress_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint64_t-query_column-bindings.cu create mode 100644 galp/benchmark/generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu create mode 100644 galp/benchmark/include/generated-bindings/kernel-bindings.cuh create mode 100644 galp/benchmark/include/generated-bindings/multi-column-device-kernels.cuh create mode 100644 galp/benchmark/micro-benchmarks.cu create mode 100644 galp/benchmark/nvcomp/benchmark-compressors.cu create mode 100644 galp/benchmark/nvcomp/nvcomp-compressors.cu create mode 100755 galp/code-generators/generate-kernel-bindings.py create mode 100755 galp/code-generators/generate-multicolumn-kernels.py create mode 100644 galp/src/CMakeLists.txt create mode 100644 galp/src/alp/alp-bindings.cu create mode 100644 galp/src/engine/enums.cu create mode 100644 galp/src/include/alp/alp-bindings.cuh create mode 100644 galp/src/include/engine/data.cuh create mode 100644 galp/src/include/engine/device-utils.cuh create mode 100644 galp/src/include/engine/enums.cuh create mode 100644 galp/src/include/engine/kernels.cuh create mode 100644 galp/src/include/engine/multi-column-host-kernels.cuh create mode 100644 galp/src/include/engine/verification.cuh create mode 100644 galp/src/include/flsgpu/alp.cuh create mode 100644 galp/src/include/flsgpu/consts.cuh create mode 100644 galp/src/include/flsgpu/device-types.cuh create mode 100644 galp/src/include/flsgpu/fls-switch-case.cuh create mode 100644 galp/src/include/flsgpu/fls.cuh create mode 100644 galp/src/include/flsgpu/flsgpu-api.cuh create mode 100644 galp/src/include/flsgpu/host-utils.cuh create mode 100644 galp/src/include/flsgpu/old-fls.cuh create mode 100644 galp/src/include/flsgpu/structs.cuh create mode 100644 galp/src/include/flsgpu/utils.cuh create mode 100644 galp/src/include/generator/generate_binaries.hpp create mode 100644 galp/src/include/nvcomp/benchmark-compressors.cuh create mode 100644 galp/src/include/nvcomp/nvcomp-compressors.cuh create mode 100644 galp/test/CMakeLists.txt create mode 100644 galp/test/galp_test.cu create mode 100644 galp/test/thrust_core_test.cu create mode 100644 galp/test/thrust_memcpy_test.cu delete mode 100644 gpu/CMakeLists.txt delete mode 100644 gpu/cuda/CMakeLists.txt create mode 100644 mk/benchmark.mk create mode 100644 scripts/generator_helpers/galp_generator.py delete mode 100644 src/cor/compression_plan_history.cpp delete mode 100644 src/cor/cpu.cpp delete mode 100644 src/cor/lyt/col.cpp delete mode 100644 src/cor/lyt/dic/CMakeLists.txt delete mode 100644 src/cor/lyt/dic/dic.cpp delete mode 100644 src/cor/lyt/dic/dic_fac.cpp delete mode 100644 src/cor/lyt/page/CMakeLists.txt delete mode 100644 src/cor/lyt/page/entry_point.cpp delete mode 100644 src/cor/lyt/page/offset.cpp delete mode 100644 src/cor/lyt/page/page.cpp delete mode 100644 src/cor/lyt/page/page_hdr.cpp delete mode 100644 src/cor/lyt/page/page_param.cpp delete mode 100644 src/cor/lyt/page/type.cpp delete mode 100644 src/cor/lyt/vec.cpp delete mode 100644 src/cor/mtd.cpp delete mode 100644 src/cor/prm/alp/CMakeLists.txt delete mode 100644 src/cor/prm/alp/alp_compress.cpp delete mode 100644 src/cor/prm/alp/alp_decompress.cpp delete mode 100644 src/cor/prm/b_ffor/CMakeLists.txt delete mode 100644 src/cor/prm/b_ffor/b_ffor_compress.cpp delete mode 100644 src/cor/prm/b_ffor/b_ffor_decompress.cpp delete mode 100644 src/cor/prm/bitpack/CMakeLists.txt delete mode 100644 src/cor/prm/bitpack/bitpack_compress.cpp delete mode 100644 src/cor/prm/bitpack/bitpack_decompress.cpp delete mode 100644 src/cor/prm/factor/CMakeLists.txt delete mode 100644 src/cor/prm/factor/compress.cpp delete mode 100644 src/cor/prm/factor/decompress.cpp delete mode 100644 src/cor/prm/ffor/CMakeLists.txt delete mode 100644 src/cor/prm/ffor/ffor_analyze.cpp delete mode 100644 src/cor/prm/ffor/ffor_compress.cpp delete mode 100644 src/cor/prm/ffor/ffor_decompress.cpp delete mode 100644 src/cor/prm/mem_cpy/CMakeLists.txt delete mode 100644 src/cor/prm/mem_cpy/mem_cpy_compress.cpp delete mode 100644 src/cor/prm/mem_cpy/mem_cpy_decompress.cpp delete mode 100644 src/cor/prm/mem_trs/CMakeLists.txt delete mode 100644 src/cor/prm/mem_trs/mem_trs_compress.cpp delete mode 100644 src/cor/prm/mem_trs/mem_trs_decompress.cpp delete mode 100644 src/cor/prm/off_cpy/CMakeLists.txt delete mode 100644 src/cor/prm/off_cpy/off_cpy_compress.cpp delete mode 100644 src/cor/prm/off_cpy/off_cpy_decompress.cpp delete mode 100644 src/cor/prm/patch/CMakeLists.txt delete mode 100644 src/cor/prm/patch/b_patch/CMakeLists.txt delete mode 100644 src/cor/prm/patch/b_patch/b_patch_compress.cpp delete mode 100644 src/cor/prm/patch/b_patch/b_patch_decompress.cpp delete mode 100644 src/cor/prm/patch/ll_patch/CMakeLists.txt delete mode 100644 src/cor/prm/patch/ll_patch/ll_patch_compress.cpp delete mode 100644 src/cor/prm/patch/ll_patch/ll_patch_decompress.cpp delete mode 100644 src/cor/prm/patch/patch.cpp delete mode 100644 src/cor/prm/patch/s_patch/CMakeLists.txt delete mode 100644 src/cor/prm/patch/s_patch/s_patch_compress.cpp delete mode 100644 src/cor/prm/patch/s_patch/s_patch_decompress.cpp delete mode 100644 src/cor/prm/rle/CMakeLists.txt delete mode 100644 src/cor/prm/rle/rle_compress.cpp delete mode 100644 src/cor/prm/rle/rle_decompress.cpp delete mode 100644 src/cor/prm/rsum/CMakeLists.txt delete mode 100644 src/cor/prm/rsum/rsum_compress.cpp delete mode 100644 src/cor/prm/rsum/rsum_decompress.cpp delete mode 100644 src/cor/prm/trs/CMakeLists.txt delete mode 100644 src/cor/prm/trs/trs_compress.cpp delete mode 100644 src/cor/prm/trs/trs_decompress.cpp delete mode 100644 src/cor/prm/type.cpp create mode 100644 src/expression/data_parallelize_patch_operator.cpp create mode 100644 src/include/fls/cuda/common.hpp create mode 100644 src/include/fls/cuda/config.hpp create mode 100644 src/include/fls/expression/data_parallelize_patch_operator.hpp create mode 100644 src/include/fls/primitive/patch/patch.hpp create mode 100644 src/primitive/patch/CMakeLists.txt create mode 100644 src/primitive/patch/patch.cpp create mode 100644 test/src/expression_tests/galp_test.cpp create mode 100644 test/src/primitive_tests/patch_test.cpp diff --git a/.clang-format b/.clang-format index 70b1fad4..f000e4ac 100644 --- a/.clang-format +++ b/.clang-format @@ -34,3 +34,15 @@ CompactNamespaces: true BinPackArguments: false BinPackParameters: false +# CUDA-specific additions +AttributeMacros: + - "__global__" + - "__device__" + - "__host__" + - "__constant__" + - "__shared__" + - "__managed__" + - "__restrict__" + - "__launch_bounds__" + - "__forceinline__" + - "__noinline__" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..925646d3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +/.github/workflows/* @azimafroozeh diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 00000000..5c6bf74f --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,58 @@ +name: CUDA (native + president-docker) + +run-name: >- + ${{ github.workflow }} on ${{ github.ref_name }} (#${{ github.run_number }}) + — ${{ github.event.pull_request.title || github.event.head_commit.message }} + +on: [ push ] + +permissions: + contents: read # checkout only + +# ────────────────────────────────────────────────────────────── +jobs: + # 1️⃣ Native matrix build (president + ttt) + build: + strategy: + fail-fast: false + matrix: + runnerLabel: [ president, ttt ] + + runs-on: + - self-hosted + - Linux + - X64 + - ${{ matrix.runnerLabel }} + + name: Native build on ${{ matrix.runnerLabel }} + + steps: + - uses: actions/checkout@v3 + with: { fetch-depth: 0 } + + - name: 🔎 Env dump + run: | + set -x + hostname + echo "Matrix label :" ${{ matrix.runnerLabel }} + which nvidia-smi || true + which nvcc || true + which cmake || true + + - name: ✅ Verify tools + run: | + for cmd in nvidia-smi nvcc cmake; do + command -v "$cmd" >/dev/null || { echo "::error::$cmd missing"; exit 127; } + done + nvidia-smi + nvcc --version + cmake --version + + - name: ⚙️ Configure + run: cmake -S . -B build -DFLS_BUILD_GPU=ON + + - name: 🛠️ Build + run: cmake --build build -- -j"$(nproc)" + + - name: 🚀 Smoke-test + run: ./build/cuda_info_dummy \ No newline at end of file diff --git a/.github/workflows/galp.yaml b/.github/workflows/galp.yaml new file mode 100644 index 00000000..1ee5a36c --- /dev/null +++ b/.github/workflows/galp.yaml @@ -0,0 +1,114 @@ +# .github/workflows/galp.yml + +name: GALP + +run-name: >- + ${{ github.workflow }} on ${{ github.ref_name }} (#${{ github.run_number }}) + — ${{ github.event.pull_request.title || github.event.head_commit.message }} + +on: + push: + +permissions: + contents: read + +jobs: + president-build-test: + if: ${{ github.actor == 'azimafroozeh' }} + name: CMake CI on president + runs-on: + - self-hosted + - Linux + - X64 + - president + + steps: + # ──────────────────────────────────────────────── + # 0. Checkout + # ──────────────────────────────────────────────── + - name: Checkout source + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # ──────────────────────────────────────────────── + # 0a. Generate synthetic dataset + # ──────────────────────────────────────────────── + - name: Generate synthetic dataset + uses: ./.github/actions/generate-dataset + + # ──────────────────────────────────────────────── + # 1. Quick sanity of the runner + # ──────────────────────────────────────────────── + - name: 🔎 Env dump + run: | + set -x + hostname + echo "Runner label: president" + echo "CUDA path: $CUDA_LIBRARY_PATH" + echo "NVCOMP_INC: $NVCOMP_INC" + echo "NVCOMP_LIB: $NVCOMP_LIB" + which nvidia-smi || true + which nvcc || true + which clang++ || true + which g++ || true + which cmake || true + which make || true + + # ──────────────────────────────────────────────── + # 2. Verify required tools + # ──────────────────────────────────────────────── + - name: ✅ Verify tools + run: | + for cmd in nvidia-smi nvcc clang++ g++ cmake make; do + command -v "$cmd" >/dev/null || { echo "::error::$cmd missing"; exit 127; } + done + + - name: Version outputs + run: | + nvidia-smi + nvcc --version + clang++ --version + g++ --version + cmake --version + make --version + + # ──────────────────────────────────────────────── + # 3. Configure with CMake + # ──────────────────────────────────────────────── + - name: 🔄 CMake configure + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DFLS_ENABLE_GALP_TESTING_AND_BENCHMARKING=ON + + # ──────────────────────────────────────────────── + # 4. Build (all targets) + # ──────────────────────────────────────────────── + - name: 🏗️ Build + run: cmake --build build --parallel + + # ──────────────────────────────────────────────── + # 5. Run unit tests + # ──────────────────────────────────────────────── + - name: 🧪 Run tests + run: | + cd build + ctest --output-on-failure + + # ──────────────────────────────────────────────── + # 6. Run benchmarks (no label filter) + # ──────────────────────────────────────────────── + - name: ⚡ Run benchmarks + run: | + cd build + ctest --output-on-failure || true + + # ──────────────────────────────────────────────── + # 7. Smoke‑test helper binaries + # ──────────────────────────────────────────────── + - name: 🚀 Smoke‑test binaries + run: | + ./build/galp/benchmark/micro-benchmarks --help || true + ./build/galp/benchmark/compressors-benchmarks --help || true + shell: bash diff --git a/CMakeLists.txt b/CMakeLists.txt index b040fdca..b3b01e81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,7 +108,7 @@ endif () option(FLS_BUILD_TESTING "Build Test" OFF) option(FLS_BUILD_BENCHMARKING "Enable Benchmark Build" OFF) option(FLS_BUILD_EXAMPLES "Build Examples" OFF) -option(FLS_BUILD_GPU "Build GPU" OFF) +option(FLS_BUILD_CUDA "Build FastLanes CUDA reader" OFF) option(FLS_BUILD_GTEST "Build GTEST" OFF) option(FLS_BUILD_PYTHON "Build FastLanes Python bindings" OFF) option(FLS_BUILD_SHARED_LIBS "Build FastLanes library as shared" OFF) @@ -120,13 +120,13 @@ option(FLS_ENABLE_TABLE_LOG "Enable Table Log" OFF) option(FLS_ENABLE_VERBOSE_OUTPUT "Enable verbose output" OFF) option(FLS_ENABLE_DATA "Enable Data" OFF) option(FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING "Build FSST" OFF) +option(FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING "Build GALP" OFF) # Definitions: --------------------------------------------------------------------------------------------------------- add_compile_definitions(FLS_CMAKE_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}") - # GTEST : -------------------------------------------------------------------------------------------------------------- -if (FLS_BUILD_TESTING OR FLS_BUILD_GPU OR FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING) +if (FLS_BUILD_TESTING OR FLS_BUILD_CUDA OR FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING OR FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING) message("---------------------------------------------------------------------------------------------------------") message("-- FLS: Building GTEST:") include(GoogleTest) @@ -182,8 +182,21 @@ if (true) endif () +# CUDA : --------------------------------------------------------------------------------------------------------------- +if (FLS_BUILD_CUDA) + message("---------------------------------------------------------------------------------------------------------") + message("--: Building CU-FastLanes Lib.") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/cuda) +endif () + + # DATA : --------------------------------------------------------------------------------------------------------------- -if (FLS_BUILD_TESTING OR FLS_BUILD_BENCHMARKING OR FLS_BUILD_EXAMPLES OR FLS_ENABLE_DATA) +if (FLS_BUILD_TESTING OR + FLS_BUILD_BENCHMARKING OR + FLS_BUILD_EXAMPLES OR + FLS_ENABLE_DATA OR + FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING OR + FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING) message("---------------------------------------------------------------------------------------------------------") message("-- FLS: Enabling Data.") @@ -276,7 +289,6 @@ if (FLS_ENABLE_FSST_TESTING_AND_BENCHMARKING) add_subdirectory(fsst) endif () - # Example : ------------------------------------------------------------------------------------------------------------ if (FLS_BUILD_EXAMPLES) message("---------------------------------------------------------------------------------------------------------") @@ -284,13 +296,6 @@ if (FLS_BUILD_EXAMPLES) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples) endif () -# GPU : ---------------------------------------------------------------------------------------------------------------- -if (FLS_BUILD_GPU) - message("---------------------------------------------------------------------------------------------------------") - message("- GPU:") - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/gpu) -endif () - # Playground : --------------------------------------------------------------------------------------------------------- if (EXISTS "${CMAKE_SOURCE_DIR}/playground/CMakeLists.txt") message("---------------------------------------------------------------------------------------------------------") @@ -305,4 +310,9 @@ if (FLS_BUILD_PYTHON) add_subdirectory(python) # your module’s CMakeLists.txt endif () - +# FSST: ---------------------------------------------------------------------------------------------------------------- +if (FLS_ENABLE_GALP_TESTING_AND_BENCHMARKING) + message("---------------------------------------------------------------------------------------------------------") + message("-- FLS: Adding GALP Testing and Benchmarking.") + add_subdirectory(galp) +endif () \ No newline at end of file diff --git a/README.md b/README.md index 3f99d030..318e43fd 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ FastLanes features: -- **Fully SIMD-friendly with zero explicit SIMD instructions**: -- **Zero dependencies**: - - no external libraries required +* **Fully SIMD-friendly with zero explicit SIMD instructions** +* **Zero dependencies**: + + * no external libraries required --- @@ -97,7 +98,7 @@ fn main() -> Result<()> { ## Documentation -- [Typing System](docs/typing_system.md) +* [Typing System](docs/typing_system.md) --- @@ -121,6 +122,12 @@ fn main() -> Result<()> { * [Read the paper](https://doi.org/10.1145/3626717) * [Source code](https://github.com/cwida/ALP) +* **Sven Hielke Hepkema, Azim Afroozeh, Charlotte Felius, Peter Boncz & Stefan Manegold**, “G‑ALP: Rethinking Light‑weight Encodings for GPUs,” *DaMoN ’25*, July 2025 + + * [Read the paper](https://dl.acm.org/doi/pdf/10.1145/3736227.3736242) + * [Source code](https://github.com/cwida/FastLanesGpu-Damon2025) + + --- ## How to Cite @@ -161,6 +168,17 @@ If you use FastLanes in your research or projects, please cite: organization = {ACM}, doi = {10.1145/3626717} } + +@inproceedings{hepke2025galp, + author = {Hepkema, Sven Hielke and Afroozeh, Azim and Felius, Charlotte and Boncz, Peter and Manegold, Stefan}, + title = {G‑ALP: Rethinking Light‑weight Encodings for GPUs}, + booktitle = {DaMoN ’25: Proceedings of the 21st International Workshop on Data Management on New Hardware}, + pages = {11:1--11:10}, + month = jul, + year = {2025}, + organization = {ACM}, + url = {https://dl.acm.org/doi/pdf/10.1145/3736227.3736242} +} ``` --- diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index e2b4270a..6909005b 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -30,3 +30,5 @@ target_link_libraries(bench_share_schema PUBLIC FastLanes::benchmarker) add_executable(bench_compression_time bench_compression_time.cpp) target_link_libraries(bench_compression_time PUBLIC FastLanes::benchmarker) + +add_subdirectory(bench_galp) \ No newline at end of file diff --git a/benchmark/bench_galp/CMakeLists.txt b/benchmark/bench_galp/CMakeLists.txt new file mode 100644 index 00000000..36601ef4 --- /dev/null +++ b/benchmark/bench_galp/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(bench_galp bench_data_parallelize_patch.cpp) +target_link_libraries(bench_galp PUBLIC FastLanes::benchmarker) \ No newline at end of file diff --git a/benchmark/bench_galp/bench_data_parallelize_patch.cpp b/benchmark/bench_galp/bench_data_parallelize_patch.cpp new file mode 100644 index 00000000..87a255fb --- /dev/null +++ b/benchmark/bench_galp/bench_data_parallelize_patch.cpp @@ -0,0 +1,230 @@ +// ───────────────────────────────────────────────────────────────────────────── +// patch_bench.cpp – unified benchmark driver +// ───────────────────────────────────────────────────────────────────────────── +#include "cycle_counter.hpp" +#include "fastlanes.hpp" +#include "fls/cuda/common.hpp" +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Random helpers +// --------------------------------------------------------------------------- +namespace fastlanes::helper { + +inline std::mt19937_64& rng() { + static std::mt19937_64 eng {std::random_device {}()}; + return eng; +} + +template +std::enable_if_t> fill_random_exceptions(PT* dst, std::size_t n) { + std::uniform_int_distribution dist(std::numeric_limits::min(), std::numeric_limits::max()); + for (std::size_t i = 0; i < n; ++i) + dst[i] = dist(rng()); +} + +template +std::enable_if_t> fill_random_exceptions(PT* dst, std::size_t n) { + std::uniform_real_distribution dist(PT(-1), PT(1)); + for (std::size_t i = 0; i < n; ++i) + dst[i] = dist(rng()); +} + +inline void fill_random_positions(uint16_t* dst, std::size_t n) { + std::uniform_int_distribution dist(0, 1023); + for (std::size_t i = 0; i < n; ++i) + dst[i] = dist(rng()); +} + +} // namespace fastlanes::helper + +// --------------------------------------------------------------------------- +// Kernels +// --------------------------------------------------------------------------- +namespace fastlanes { +template +static inline void new_data_parallelizer(const PT* in_exc_arr, + const uint16_t* in_pos_arr, + n_t n_exceptions, + PT* out_exc_arr, + uint16_t* out_offset, + uint16_t* out_pos_arr, + uint8_t* out_count) { + constexpr n_t N_LANES = cuda::get_n_lanes(); + constexpr n_t MASK = N_LANES - 1; + static_assert((N_LANES & MASK) == 0, "N_LANES must be power-of-two"); + + for (n_t l = 0; l < N_LANES; ++l) + out_count[l] = 0; + for (n_t i = 0; i < n_exceptions; ++i) + ++out_count[in_pos_arr[i] & MASK]; + + uint16_t running = 0; + for (n_t l = 0; l < N_LANES; ++l) { + out_offset[l] = running; + running += out_count[l]; + } + + for (n_t i = 0; i < n_exceptions; ++i) { + const n_t lane = in_pos_arr[i] & MASK; + const uint16_t dst = out_offset[lane]++; + out_exc_arr[dst] = in_exc_arr[i]; + out_pos_arr[dst] = in_pos_arr[i]; + } +} + +template +static inline void old_data_parallelizer(const PT* in_exc_arr, + const uint16_t* in_pos_arr, + n_t n_exceptions, + PT* out_exc_arr, + uint16_t* out_offset, + uint16_t* out_pos_arr, + uint8_t* out_count) { + constexpr n_t N_LANES = cuda::get_n_lanes(); + constexpr n_t VALUES_PER_LANE = cuda::get_values_per_lane(); + + PT vec_exc[CFG::VEC_SZ]; + PT vec_pos_proxy[CFG::VEC_SZ]; + uint16_t n_local_exp[N_LANES] {}; + + for (n_t exc_idx = 0; exc_idx < n_exceptions; ++exc_idx) { + PT exc = in_exc_arr[exc_idx]; + uint16_t position = in_pos_arr[exc_idx]; + n_t lane = position % N_LANES; + auto& count = n_local_exp[lane]; + vec_exc[lane * VALUES_PER_LANE + count] = exc; + vec_pos_proxy[lane * VALUES_PER_LANE + count] = position; + ++count; + } + + n_t global_idx = 0; + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_offset[lane] = static_cast(global_idx); + for (n_t k = 0; k < n_local_exp[lane]; ++k) { + out_exc_arr[global_idx] = vec_exc[lane * VALUES_PER_LANE + k]; + out_pos_arr[global_idx] = static_cast(vec_pos_proxy[lane * VALUES_PER_LANE + k]); + ++global_idx; + } + out_count[lane] = static_cast(n_local_exp[lane]); + } +} + +// Thin adapter for Patch kernel +template +static inline void try_this_data_parallelizer(const PT* in_exc, + const uint16_t* in_pos, + n_t n_exc, + PT* out_exc, + uint16_t* out_off, + uint16_t* out_pos, + uint8_t* out_cnt) { + constexpr n_t N_LANES = cuda::get_n_lanes(); + constexpr n_t MASK = N_LANES - 1; + static_assert((N_LANES & MASK) == 0); + + for (n_t lane = 0; lane < N_LANES; ++lane) + out_cnt[lane] = 0; + + for (n_t i = 0; i < n_exc; ++i) + ++out_cnt[in_pos[i] & MASK]; + + uint16_t prefix = 0; + PT* exc_ptr[N_LANES]; + uint16_t* pos_ptr[N_LANES]; +#pragma unroll + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_off[lane] = prefix; + exc_ptr[lane] = out_exc + prefix; + pos_ptr[lane] = out_pos + prefix; + prefix += out_cnt[lane]; + } + + for (n_t i = 0; i < n_exc; ++i) { + n_t lane = in_pos[i] & MASK; + *exc_ptr[lane]++ = in_exc[i]; + *pos_ptr[lane]++ = in_pos[i]; + } +} + +// --------------------------------------------------------------------------- +// Benchmark plumbing +// --------------------------------------------------------------------------- +template +static benchmark::BenchmarkReporter::Run run_patch_parallelize(int id, std::string name, n_t N_EXC) { + constexpr std::size_t ALIGN = 64; +#ifdef NDEBUG + constexpr std::uint64_t ITER = 30'000'000; +#else + constexpr std::uint64_t ITER = 1; +#endif + + auto in_exc = static_cast(::operator new[](N_EXC * sizeof(PT), std::align_val_t {ALIGN})); + auto out_exc = static_cast(::operator new[](N_EXC * sizeof(PT), std::align_val_t {ALIGN})); + auto in_pos = static_cast(::operator new[](N_EXC * sizeof(uint16_t), std::align_val_t {ALIGN})); + auto out_pos = static_cast(::operator new[](N_EXC * sizeof(uint16_t), std::align_val_t {ALIGN})); + + constexpr n_t N_LANES = cuda::get_n_lanes(); + auto out_off = static_cast(::operator new[](N_LANES * sizeof(uint16_t), std::align_val_t {ALIGN})); + auto out_cnt = static_cast(::operator new[](N_LANES * sizeof(uint8_t), std::align_val_t {ALIGN})); + + helper::fill_random_exceptions(in_exc, N_EXC); + helper::fill_random_positions(in_pos, N_EXC); + + const auto start = benchmark::cycleclock::Now(); + for (std::uint64_t i = 0; i < ITER; ++i) + Fn(in_exc, in_pos, N_EXC, out_exc, out_off, out_pos, out_cnt); + const auto cycles = benchmark::cycleclock::Now() - start; + + ::operator delete[](in_exc, std::align_val_t {ALIGN}); + ::operator delete[](out_exc, std::align_val_t {ALIGN}); + ::operator delete[](in_pos, std::align_val_t {ALIGN}); + ::operator delete[](out_pos, std::align_val_t {ALIGN}); + ::operator delete[](out_off, std::align_val_t {ALIGN}); + ::operator delete[](out_cnt, std::align_val_t {ALIGN}); + + return benchmark::BenchmarkReporter::Run {id, name, ITER, double(cycles) / double(ITER)}; +} + +// --------------------------------------------------------------------------- +// Register & run +// --------------------------------------------------------------------------- +static void benchmark_all(benchmark::Benchmark& bench) { + int idx = 0; + for (n_t exc = 0; exc <= 20; ++exc) { + // float kernels + bench.Run( + run_patch_parallelize>(idx++, "new_float" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "old_float" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "try" + std::to_string(exc), exc)); + + // double kernels + bench.Run( + run_patch_parallelize>(idx++, "new_dbl" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "old_dbl" + std::to_string(exc), exc)); + bench.Run( + run_patch_parallelize>(idx++, "try" + std::to_string(exc), exc)); + } +} +} // namespace fastlanes +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- +int main() { + benchmark::Benchmark bench = benchmark::create("patch") + .save() + .at(std::string(FLS_CMAKE_SOURCE_DIR) + "/benchmark/result/galp") + .print() + .add_extra_info(benchmark::CmakeInfo::getCmakeInfo()); + + fastlanes::benchmark_all(bench); + return 0; +} diff --git a/benchmark/result/galp/benchmark_new_vs_old.png b/benchmark/result/galp/benchmark_new_vs_old.png new file mode 100644 index 0000000000000000000000000000000000000000..e841fad418c4553476a9a2f570626d4f58e5cc6c GIT binary patch literal 77077 zcmeFZXH=72*EI?vMUW;UAWcQZLT@3V7XcNbhzd#vrAR0AD!qeXD6so$ZXpi!ovpxmIN0aqTcPTl~2!Q8GHxam1rxp|tqKBmwycXPIPaUOf837X&b@-#henT`h`_aqif-mXdsM%FsypJ@#D?}%AyCnDZ zkw8|JyDY5clnue>UK=djdBd^j;JH8VJ1vxC|2ciM?E3mHroG)tBGr#D)`83L9It^~ z77!4iQl&c^%JN@-xTw39QUB*9)zH&W<$r&TDjln58|AtGxD;8T#>Da;cT#QUb$$Au zcRIK7j7t4KFS$hUocXU$C;ux_Zia&OKQ5_{oTZKYw`T-DoMQU>Na~$QX=%}K-kjG? zlR8#XpPQigWW#a1R$=(g2_}>rR;fpk{>4p^%A*Rtn;U~Dix&Da2kfZ}H(rRf{+UU- zwYpM;Ppca!G>%cvdEAjam-uRYNe40-nkH>$RXbKh_`qv;0or>-H5f ziX>d*LJr~#i(o>MS_ISN9tjV@grB<>ovji=Q$bp(nw0tW;&ObDF6pESd<~IrrE;Fn9*gSw=(E2==*|_^iF~`>DcL>I$RMgd`qf>soLAn9 zWjuf+?6a{DRO?Tg`m z$|!dXhbS61RGKS%?cnn(kbb?V&K#_^Mabqd*7u{?e6wR1J-e8B<544@#XQ}gr>^40 z>&D;US=XBSKN*TU3`a`|0BW-zzyVDVWv zboL}$M*mm$C7X`J6&KN9J&cBPx124?ohKXHUUegMbbNfiK2Ql*Zx_tHoui7r?uvCN zHwTNxM-uUQ~InT`4PUFX?a;{>ynkxE(O2%@Vp!)q!kMcOX8 zb$)-3a$~&j9()2=J-^k@!avGDzFEPFZlB9!y2P=iVi2H49van$%ii(zLqFZVI z_}lxD{w8(%dsWwOp$YHrhyc)UQd8^g`pRDX?rC zTO*HO8Czi%`)(<9jco1`$I~75zUS~N_?*nU`3l_y8As06fEZ2*aLs=Rru|;~})nGRxB+Ehl62O}tx>5K^w|?6-x!f&?xie6Bmo3H|W*ub+~nxH<-n zbXhyASU!c}on>vqS8VO=?QLystAl8Ko3K;$TDYy+p3GKNzE$9xyu8UET0Z5LX!bbE zV$0TegSr5J<#9ysHIB*BZpDt*4-WUf;NQFu`E-jo9@wQWua6QnL>(_550C@lWqpSv zn^PK+R6Cy)sHhQl`tilu);R8n7}456?Qppq#B0NHM=J?J<$R_j+ZeKbcFkdnw<{e2 zVVf4Z)MY6_&BRN^En~kb%cr-cohs3y{kAl-!E@X{VyUo7q@%KjOKqNkm)DKO`8Dy^ zhaz+Ye}GM9<%!Hzi)NbNV8=oTr7@?38ymwIx&lONw_f+I{~j>qraj7gYOH5lrJ(uQ zSvBWzf8w@1_TGp4lfeuk%4<_CIzcw@sU-e@t%i>tf!njMsD24#>Q{Lzcg`qbliYby zhT9gA*-4RXUEPYLk1jUF#&Vu3u^c8|l={Z*_vQlZ38n$tEwY#C1k)ZY1+&d!t~=~? zIpCa3XS%caL4=A^*C+5mr`Gt&dlwr~=bdV|Fa9Co6O!PtcG=z(^gG>L4Rrf+UN?&3 z-PG4{yIn8t3dv(in%AE5e(Y^IsD*Ax=ZyosHCW$K6&`!T!I<+stRX%)Lx-Wo9N6y5Dc0}Bk?QD6-y+HE_QNsQ%^ z41T}Vw7?pm#Cv+j*hTx%{EzI?=S=70bvH@+rv8t4uEQhZl`ya+q3JWOCRY#7i8j^n z!xTQWijl)RN4Y0!f@CQ<=1(+E@Sg|qHpUHcM7C~nNm@M`GRf6oEOi*G_D;p397mtT z%h~i~cRHwr9~UR-WXeNhP#YU=1bJ%=PwB0ua7IZcensT=-do+=&P~(6U1@|NSi(-! z+H|Y6BZ<(h)Pu+*wcBA!=n zu4G>zLEd5vi*hhT^9SxE>Q=hW#-ii6t#KojeU?h3>WV=4;QZ{5demEk5(_k#w9Lw7 zuwfzd?X|A!!*)ff-J%A?_0=Y2Hi_N}{(syEK8xS;=60%iT$e9jwi-v8?Pt38KX`yDdQgh?+e@|1 zTpM&uV5oIU*gir{{|Fp`G6KTF!iYLB8$&qaCf3PXZZn*bJ2~;ii9*|FyLeeF#7A@DqSyr`SI)L2g!2(owe4BuY+`tj(HVq zPF~q052zNEuq@A)b4u>l(^a|D?=iinV(Cjt_B{eJ-}&2Dzyd+;yZG{?Qg>tL*amKh z4a|`ub_1-vq-{kCU(`DkC*^dgE+ewkI#l`0YBiD-YR`~=*7cV)|C&4(?-Wj;Dmi#m z6D(Lb6oe!}LO=oc!&3g1#<|dKA!z|mm+#6*c-DS?TcQxEd z*QJd`IcsE!y?@H8uB|pFO5H~ez;k9P_rHZNH!Vafo9+7&4IRzuBqgjS>VqymM?)Rq zvXe*I&Q99l#9>jp{bja#eauSxJBUnav0W|!B~HdB@ou`e;8=ZBTW8yMkzk1Er(K(C z9_s$<6Ww|#)51vRr?}^`IYl9W7&QX4A$RO#h35Hiv#G1o-h`B&G6k?(G*PF zZhN=E-gYWVxor;%D+hmv_?Ed-PB!{oIypQWs zQ5HNVXwIeENFbsspNwPRC4i$`CVhPa<*;)Olp zIm4{Dl^yjxTP?b1`QZg9RGAJN;V#T+TOSz{#U_|`>7)ixmiH9Wjf@9%6zDbfi)Ceb z;U8(IGz|8eunnh}7ThczvGc;+?iVi`$Re4J!>;+o)hL_!v-6Rv2^h(u#b=iMB>}V< z893`xGS*7ezanFgdZhPJUOvp=O8;PA?>PW6=2Clk-@r3q7xS0}ZG1&5{yYV^_mQW7G@S$hV&}1UDOt1 zTSn!EP-7iv>2hi8HMKleCAFQJr3~CYc4jJIW$#u`zER9o-J0+$&6dp&6MAC$_V(GV zRx8SNfQ?QIK9!`O0jR6fZ#+vae)FkA>)3M9lqMMEyE{dw7r{8Y5@2Mv3KW4*%`L8~2 zPbVnFQk`Mpq;9r3iib%CsnXJRC)vJ5PDlTiN$GzNYK_!~Q3(FME;=Do`1uoU_&Cg^ zd{^H;iv?X?2-r7`cplr~FkSfGaL*ig1R18h0TW+g$qRhm#8J=PUtZu?E%wBlFwdV` ziYs#(SJWI;+TBR%eVlOpaNqdl&>dEctSs6wmPK!^Ga|u}oz6Z2-fn8OOAmi8@v^Vj zyh)q&aX;(CQ5ey3%>mx;dO{v@oKJpGJJ!Udg}q`n*-%@>J)iP*>C>c7wfB=Fl5UFD z{Lr32tiMbmfb!>4l#lt9H`U6m6fxfR%2~ju{W$#Z$@N1`)X!3cVpHN+Zx->CD zQa1F2?so;x8_nLDXCx(Kp=;gk_IKWBvG$F*c@&#O@e0@5Fu`tY(fy!QU0BW53!ZP1 zG8+3pgqj$1BGWj-lr@_>MCf+vmt^8I3jP(bNZh43jgRWG?sJ24fd*si2c@|M#o9U3`>3w zg-2E4Yq7G$Tj`cFQAHbuILXFyt8S}3-i;?8?bT-04OFds?btXLXo;gbC2d`HOwY1p zK_Rfbnr4-VV{ilHbzDO9Nh4+-a7qzZ^QLplvFg2PN|ib@c#964s_(4{v?a#qFWOLv`PQ+&gkv@aG{QT3_Jgu#n3clL9F7VpE%g=eW%hd~* zc`10Nci=Dq&M1~hs?hk;8rePGFF0uFau6}`bTHqQY65(kbD|{E=hr*!2&ybDg{IBy zrr^2;5#z7fPUE!?kWaO_(GR8fN#pymh`7Xadpu`C$J_U;A42%j-!(5#SSa;ytpNzd zt>Bg>F44k25`mG*_sE@o&cqJ80%`*DEE3^Qz091a#EH;4TUdZBUug&imfw=wbV!9n z;9&ME}no1IC6Rm93vsSDA%*_W zBOR_c-P!*Yb@qQ|>A%g=QRS{QX_r17NP}=SAmg-d+-OWt+M_F1_4g-jUK4tjmiA-= zo+fp9_)aMEAS6NYk4-$ug@5Isz-2w=6?prjLCLcr$r7`q_1cB+H$|@r6?-f{?8^jI zzkOL=Q(W!7RXKSZ#g87W-|x8UgU1pW#&vYF17zE-t`bDZt!+J;D@Ov_yM> zy4`gYJqCTM8P5|1E_3RT>Ah~0g5OqgW$&ywhlr64O8hsaK81v-u##b|pSO`)?=^#J zuM*9y9WdjAwf=Ih!M%$1+)r$NzJJN3dGGz*hDcTczhB-niJG^|9gPX2fSheoAdXm~ zux`u4?ADpy`K|LJRX7R6m{Exh;eZn4;i#R6){I|DcdCVU`1)?*#QiZY<^&LHskM1z zJ}~uq2*6PDUVDHfyOzMG0QgdBL?<8>c>(8e?TjCx3t}@;QL@cLBJ*0z3y6a#smya< z>PO4$0vcfru)tf@qGs=fhI6#jTP~d#F7XaS5viEv&FoG2lHxgro5y!nJ|^249TR(Aj+i`1@K0SsAvXR4(Cu*FCM$^b0E+L$kX zTl+G47f@JEINtyW-q@xNKURYssaloW$~ImawJO5`qVma#`-0o+8vF;vXk%aJkqQ^| z#k@~A|8Dap{KC%<9P}qIJ#9}Cv>vXlW(KjA3FQwk1q{yWPkqzJ0r8|B^mv&qo9nr9 z;=O7}j$;61*#ebF$XqF`T9HUYF839gDj~+SyRAwecbqC+25G>j6&tGF1pTk^ddkI&Q{8_QiMqW20Pez}tA zHFZqbfKB!2!FftKF&?-t&DM|6^ysO0BfbY{qN#Yfm3k0;QDk79o^)9+-%I;c-S2)w ztJl6;tCMpyj{xU1M`PRm{DKw#=_XX-aNbRhYCu2FHMHZ9**V?!Y~)3p&e&}9wRxO= zC4Y*SPpbM|5X1@TT%e0L){3Wre(9~xPyCjC3C1%9xS51qSz8@Vinxb-Nx$#d6bAxW z%|}`bk61?h{;8jS-5=Ep9ttpC9*>}}J`{fa##w$#X;%uPKs2(%ty?{1qB%Xi!w26Z zpv?j6QTfDD5bWXj)Z+O#=I%JQ-7y|{uO3;Zg)K5mzG(No%G0S&Up|7&CEMhhSpQ0(64W^c6Weik>XLBGd&O3 z5pf4qStc(wBoZ0BO_;Y{X|1bV1{E#w-Q~c&9V?;l-m^76@z@17^Q`BfMCu#)k?HpR zdVZZ3VeqZ4075O3*WPlIt99Oez&HZzA@oQrAzZwG`vV{b*&VPbX{M8#5T}Vn9@o9i zg9A{toe?r(mRXZQS(atn;WCxBDIyqa`)V~&-^-~X>|WV1TWgndwE%yj13sJIrO#k1 zBW)SqQuzJC<%;uq%H~@L7;)Z%PD%SgF#?45qzAnTckP___fv1!B8qJBgeqiDYGah^ z@M4+iVtqE_d`W5@ z@R74Uq$@Bv8fHFwoa?WA;|^a~Kr);Iiyp5;6lju$H>=CzZpn)5G;B`Rob0Re9xvYo zBx$OQV(SQ*68@7+G^p0hU0zu>BZL$Aan@r6tDX%%T-=_n@+`KsGd;23U*(G@zN#L7 zWJ*2d#;O`=-_D{Md2|j)E;$AD^G(-*ydd$_6Q8;511K%4Eb=NkVnKXpe0umr;z|F< z)WqauD-;Mkq3PAhCcf6;FLwGGfX9?@tej2m1dHS#+3E|@qH7-MV=3mFtm}-{+$O7l zM#{2Y0le$S-OrEOV*$+f@e3f#b#VbPZ*>qun6?HQ)~^ygAXd$5TdlxBWmUJgOS0~L znVFfn4Iy5>ovn-pX$tV9>m8z2Bv2;4sEnKMbg1$9{cQy8@!{c@CmMI|+(Cn6x!mg! z#=x<;5TlIs0+3+N7f^Dcu^=n>;FJLAOc>MT_kT3}@mkC&E}$BAZGi-@uxXk8d$CWj z!o?aum5!Vo|aotC5vuZ*7SlBIvE6{9A{0l+xhQ@r6>rcilO0GcRd0g@(!$TY}-8VJBfh}QLWKhyLm_Fh--XYuF z?rqvU4Kx~21h>Ny7_-&!(^8ji`uf`V)hI=W(ZLVgze!I*Crh0P_ zgU{In)9Y^5en4fM+qxyc7TUV$IQlg;z;lfIPgUz4;8!He*G9Ihrad-;$O!Hl(6*00?akFp zzb68cympr2Xs93J%X3BnA$kK56bJaK$WtvGFBVm8LplB=UD_8IL9!-uVNoL zH{z`yQR!=Lhg{8lEBEZA)nPkJ6t7fzqki-m{p18G0oPqJVzenwLlD zas421H)O90;J~5TGKH%abyl)2I%*RuSc_r8Kqh`ZBE{Rk}+hYvx;1d?#bHMpKJQW#iNrGo4tcj z+i43pk4g#HB4reFBW3&TxwDo#NEBGK^+_6R ztS)<}w%JntayLm2zvp@f$eD9RFjQ^_x!xvn@F>vU)NSvON0Blgv>O`n|k4Ixxk z-O1`r>(u*cL{%AiPmLIcv+81R>Ate4QlQ(u>9D%Z?dPqnFbbM46H`rJ7UxqoWkBCK zMu;bY^h5nA-TO&K&+WOc#Eju@GU3Znu1*WF&K3g%=f+}dYo=i!Ii`-FYwu`!Pb0H8 zJ>>&Q;GFi6J%+Ctl?%J?I4oP`VVxq~=e7%mdy%;aqHWg$w3sCNr;v-Uj9jBCRUVH` zSF6&75q0RXHee^D77Zn6m(F&(G~K-&3gISaj^lyzl1^n_0?pjq`IjtxXkbyUu1|Xs zHLO|fyv1hO2}Fro64(L2!at!I8zDfNPd={Bndz#i+juCm6X(3+Gj1jqs8(y{Bbgt# z%0~)?jBPeQCD$zKZzs<&_CH0a8lthMX3k5l!6Dq2#yprDt_?pJlTlp06WH@WIsJRq zCx<%CDkSk}*_yY=9+EDnZ0*>xEq-|{tDq^^)@5pfwv6bL{_;0lM_N_D-MZL-IOx?$Z z*FTCkm6>QucZQknh!n(iOQG2W;p(K#eAw6&JO*RVSxjY{Cbdg=dlK;t-Nt%B=yp#;^n+;`!jZk8RwZ!h6-y zPizlEo^{Wo;xewbS5L+xX@j zJM-Qy!Q%tON7mC=>R-!gp<~5XG`z}Ab-SpN)CZ=kz?~q~?S0`H>B^=+2LIllXYpLz z!vk&xt0v8;wMDuqk6No`RX$M`?yPtw)LDVsUDnNvMUGVw?aNwRIoI~$Z`R^JD>HO>s;*}_jCu5s#P*ec1*y2!FeL&YMvLO zS@mW0Svn4TMB#$PjSwW7-pPk^P6?P*5J`{5kFKFHCcNmM4t|(p0pareIx1V@zwU0dJ*p9d#^5eLept$4;Esw=l>P%q155_%n#n^;T{p)64XJyN5)nYLIa@ zR&Ms90~h=TR+sxfvO&wc*rl+Y<`x%Vzmy?)i(1()I|A zJCd$2Ia;O?2OpS5#bDI6#$)d5nPD-dHc65<6a+bin-&~~x3mY)q)TJ8Q0x%AtE0@o z&)I%;ZywZnkZZVI=n>l{0RV!>(e5HxpgL8i|)OC@Vq z&PZdKmV8z#pRdsQt8Kv-dSsYR=9l@bwQ#JFRq+kDjP}`1?$@5w6GBy*E zflhDDW{zcW0aJrjGBJfI+Oj*`eEMUaxZjpDLcqFUeM*N-&#VNcapr0h9WPrabQdrb zsp$dn!qztOYj908_l5Yt&4?oL`5i`J7#I@R7fa|~I9+J)h|fB3x67qV{^wHvr;ZWK zN%LxzQ{-xwPTJ_``qp$pdX%NlE6kn65NH#E6{F^KsIOMmWEDu}62@%Y#~?OP z*0wRO!Q-~meT8{(-2rsq55n}|dbmdkVeFY@!n?O7UX#?hEpW)qs?6T2JOXYTUUD8g zD(RU-npmGn|C?|C#KV`LB2Y1++*$K8J31`CtOBjdZDOo()}% zAoIcJn4Mb~^#}W&S_VPjQl(5Sz`NG@0A~OuU^05|&kvp+o95-9;fMYA76hGz4x_IS9bph`6D?>TX}g zeCw$ix@G>A)^-<>l$=j(z7!&6?Tqq@4yJH)AKF4)RN#osl4}F=l8yR_;pae=tgaj| z8@{Sx0=Ssrh0YH(nu5#%S``yf4(f(4R%?U4sMx5LE(h+cXtU?>+E+V$&-Bk{*K{qM!ILNjtM?~rSXxi;-mi8_gCg>;*L3>Ccx*fu zB^wD#7oz; zKAo2IL#psWX)uj&Oc!)gN4qXy*Lgm}^K>hXAQIvF)cJMlRg3WBTo(6_7$dr*(c+WZ zN$P4T5aEw@4e|{}B~9GtPxn+m_{LD}GO_)64n5Hj5|{LpZ5347*WR@2aL?0>l}SuB zc?7S9CnYUQO)NU+^hH3-++d>tq-8q6>h{?ukV52Cv~u`zUvI+?(`u-5zLB$r?<#sI z98Ul1@t@EUS>?e3p*xiUyMKg0@^BiKrST5A5y9s4Y~iHep=^{9fa8a;Ey}-XilSzM zq6Q0w8cJGgYk^g)b8z1C`{hP<@2}7kWcVW1gDTE_2o>*aw}PFqfsh_%H)v7Alx>*5 zi_9sOO)pQ+{A9}0X^L_YX`Boe97cNl041Y>K;~lcgaYNV*s!l#V~=77L+qZ}8%n&B z4Hbc82%EhnMzZZi+(uvIvzF{25Y0EUeq}>B}!$T0A<2-U7h=54#<@ z+Vto5lJ1u$KeH1AumI(e>y@pt-C2q?D*eDAMqfZbtLtb!e0Um=?O}laJ#*p0HP{<9 z*c$;WDXj>%!(X8>z@W()5#~@z8)F~1_>5`sS^QSd8Q$+_$-mYLEN?wS1>T{t)C%nm z7uJ1!bPrVGZ^w?7_;Nn)Ud^-#pS}t1q-nqC971#OX7c+DN`5{jk*`O%HAnn&IZ}z& zH@NKQy`u^=vK~0jk*986=Xf)vX1y+?K&QRZn3+O% zwh0lxw_Kkd)jetjxST;ytqj&hRQ8k0?YA5?$L`Z+@Pk#6h<|409K&{vJR~A{eh!BF z&zLf`G%Y5QH(&m1;3IiKTK^hx&QI3p!%uth{KC!GfXp}F&U|w`GpfzqeH$d3)F30k z)}Oq5=luH+@l)C5t6712;(RrLQTtGH5T)va_P=fTZyh}P`HN|i7+yCW9$9A!z_3fh zp8~N7*>j>lPTZ-G~hb0c@ z$1z2o>+J)!%MQcY-$p&tt!rhlhkK3X9NqI1W*7seXLN|6$FFjXt2`9P)0F~ix!i8P z68Sxdk+;jaRNn{8>DlqD^t=s?o90(O+4$bcQ-h5ZHZlI4h&KXS3o{G-j9_dC5<0Gb z^OCH}$am4TXv1OsgNPKkFMYTHo8L3xp_cRVAM2pgw~H@FKrJDiw6r(x(0oXZG`jan z$!&q9|Ko!Yu<>(%E>Z~-QFsFQW+(_Yr@NmX{+zBcbgZN%Gef6@AHG;JviI<3gs^E> z9V4fM^Cz_tEn3V@^AU_r-)wFcKA;M%&@%go{pBc|c`!d(@>143nJ9+-`w9LW@K_h% z(S8kVi(Y_rS3u#h<;?v48?0n6K0vxOfD6+EhDzN}52ZJju>;jkdZku6@_Sp;6>Dg2 zodP1Tj~usI0k+Bul3GuGybUsM3g2e5o=V6EI?bFMa1vR@PwvF0JAOTuA?Jw(-fFw} zc7MmZ-OaZg6FGH`#Ff#x2jsaK`Sj1G4K~~J+s81gDl82ck}jClHD7hKH9gqm@diH+ za}?_b7!rZr*#k;Z>O5{PmIOS;AY>!lFnrglSP_KCn`Nc{pX91i?Bi}mQXm{(olZzrJ0JAtf{6<`?H%OJ_lJ2CjB`@K+FtJD zP8U}opt7tmgT9IounR!d-#lO%nDN+T7-}u=fiuX34kOS*XKA%gnhltw#r+*f_U$bhUFOC_{Rs5 z)9up)K&np@NUaI{PVp_4WO{SXK(ua0Vk~gq9~f{hAe5Lq9G!ZygVt2nHD!RNCsrPR zz?d9C;5b)@?dr2tf|iy1VLcV%w~P2zd(`Tk>cR|j{%%TY1lr9p)E7hMpu1f#6+l}; z>rUODp@=%d!^882T*Z6WxX#|t>kQA)1|)RMS_=Es`;3MAgKZiikP)_fhHV*_7i(3FoHmni6|FL%Ddtbd21nBtrrvZyP#mxVUlKg;xj{T z*I55L>Vb8zGFxwOz7WnMY+5^lvz+FMoFQ`+AsOCdOnmZ@SBt7ofJPXG8y?XX?aqw!k2U^XJ#(L1?2y16u9 zN_!E0)T>n7{7BC`w)%4s9m$l;O2>Z`F(ciRUtV0o*G<1SonkL9D>^(qKR!9UZ=^f# zCUOlZ?w>Ox=R)9#VA@07F|sX_3#$<8&dumeqzmxzoVXx6xNzIdhP5f0bE>(aQDqy5 zu(^PFo@-#qBRl-%IVKn#WQr@g?&%wckp-`rpy`CifYX1lX*XD$vmGQGGb052<_kHH z9*!S#cJ-R4pFB$BS3Cd>Rev+1mL^kiq&IOTImrUUlWnFlG3NWphG}lSX8nfQ_w(5| z*t@&$eR|@6AO7iM{E^f7_fLu#4hc}Cp+#y_Jg?SIwbUyRi_e`?3lMJoHrFLS{y%2l zuS~ueukUZJU;O=|4y0iLBLw>T_3KvV0Uii+$ z1`ZJ^YWszD%NbhQ$61R0pB4BBB|0f$l!1HO1>46NX2c%f1A=HNuEWm-NJOe2wXI>D z>QaNHg{%<3wwJ9MMz5IAJtgP3I-}bJ+DCE=0Wxb^hK*$ZllpdwJw;b1Lhgs6een+# ziwbM{qxZc2*k-FoO_RR2mv{(&uiHEAYZ5prA98jDMQGw44k){7-AN(i@cCC%nZ50` zI3Xjt4ZIWdE~m@-TzY*xulUga!vtWs`i?#CLy8M1<-;@5^5}dn&Bt1IXJ$+zU4+hn zd}6NE2OJFAwJ3l~(;a}FH0`om6Xj5~H)49!7j$Jwj1y_?!eTka8)v-jMk>VnL1!9h zKO=1zoVtAHDCiQ&Tib9k4OcJ*{@Hvl9S?QVxkqvqUT3EckXwae$Uz83xNTi_HYpcb z{h>z@!!F~*w6hxWawS^#bgkBP6^)ovTl2vTouyGYx=`2Ir&!0**EePCDKz4bV&oDu zc*T5WdOmQ(R8zh(vVdCb?LtV?d3k3oF+4nca{cmzFLc5#&=qh(zY#D8aP#m3HRTuR z9$~g(B{B7&Kc*38?bpJtpR!>nRL|G(NbiAeDEFzvwcjCA6r~K`S9d{*;=-ph(!PfP zye!2W=oQ!vTEd=Gd+nL&@;YUD&v7yH9w3f$6gCk$Sxm?II==xl*Fuxi!R$?yv}nxtmGt%I%H78Dzac^(??UWai?S3 zoI}<+XC0FvFuk)Ho_O&iL*t(_+Eu>?EtNxP>L;7(UtF%gr%!p@#rk34JG~%grK1Ly z5_QkRWgN>;eb9fJBt_Jjo^jSQwzQ7E4r%cO&KL7c@*Fme{slQF9@1t|cMfzMnKx2t zS)hmU-Y+bpN6vTibMZnOEW*{oDeCr_cGekI>#3BNhbb8N;VEWN+rmn8w8a{FOvX9; zyLnh-3O|vI)f0RE75oBxZWWLD`gLHzXDeKl5p7>QTj`8hm<-R-@^F;;a)V(eF z@mQn9l1TB-QD^ysIZO1iS&@`?;Q08dkPcK7`#;E@AQQhma{WppH70kM&$#MH(#6zGF$_PqMmT6wAYSu}^*scEoCAMBATw-4f_dp8 zZ^N?}li_cyM9x?JS|7TX$FZk-0EEw7%qHX-LO#?+D5XhLW%{3zB^sSS9{V&^Y5a;8 zCXHX)mp!j70cYsOb3y9UaqP5efc!9kPn5=5N|Kl`{ zyv~c9v3{fGA;WCv->(H=SsDlUHEA_jP-)%#-a3dr|1z&vbzc(51LX0Hm&iJ;fg9*w?^60^9+) zd;cv;z{kfrJLh|^x5V?-L&hnD4K;4O{BuF&posntH0jl&63?R$OwG+V z`Za`{G$kZ60~ z5*^7>@Fo31WP1VVqO}7?8__w1IRnta!x;c5MM7GRnLOHiKI7H)Hs~Fp00uLL7!UGh zjf}iqXkQ6VQmQ2ZZw)uvDcI3oEAR?MJ_| zoBFL$2mDzSM0(GqR;)HMlpyBedNy;Q4A6Y?oM-s6A`-S)Y%$rB;&*xH^o&lq)1!$^GfjIF2Q~qVP zQyRDU`Yi7i8Vz|78joS8K?12{ntaZ*s3rBUvxJomR=dv!&TBK0N4AH98TxoV${Oab z`tZpW8PNy9(Q&8GhSB)mpnt8>`1@w?)c+JI81HH2TDdy}G>4J1%8&WDT~GkhO_TS=px*kZ-qWi<>noO@Kc7mhG`-;(z1@)arXc$^?v~#k)Oq%0eM3n7khNja)w{BL ziW$7Zh89jQue{Z=RJeg*J~&s;s@tCc&!**K#Y8|icM;Gxg4oSvWi`KWzl7Rk3>Imk zxVd^&w1>-VLqLOQHs}c@e**y-IRf5j3fMWDft4hslAm@72uBjM4nX(s1-3l0o8zz# z`3I3q*-#WS!5NXXk)~gc@2h61MK)A8^;U4yMnY<*TMGVpSj z>s68F;SRE(GnTE;4H!~$0YP8!=hp|aKPk5{Fp2U4?mU|XOxV}C{Jk9_}c#HB#LyR*qw?o~zA zms3Gq+!5)ZDfsDJ-1RcB)QK-xmuH@s6kn0&egXyA_7ip%k zRAc6aNf8ikZiDwGhTf7rZr*4zSk?PD3!KSP_vTUqg;+o~$uhq8XXIIOX888foMqpb z&fB}yC(`tt_*|Gx&L6vX$R^E1jfS3_NARcBp3Q0u#$je@2TcI-j!LyNa z!v{AU);h2HH6i5cSCc^A35zryB)`16eQq0Du9m_nB0RpiM6|%hDgwC!*^De>>*n_B@BB1OTrClQVreS zNE(MRJR=ji{s6=N8;fiHy~O=$&wF<8O&iLSjrD?A6g58AOf;YUFR9;^9{{xt9)Gn! z7w+dQ&f`pOqyaRE6A!Npxl@fytbxR62{uFpam-gI;~6j%rxPE%99!HuvuRg59|Jn> z(jyo%E`;(aVV7F{tUG-*jB$P>^SXIEJxQg03EgA6lQn_a*kvP+J(RDnzk@J}X?_#Dyl6S-Nx9O}ZiY8hFtGL5 z3=-*o8V9BByMsxENvMZv0DIDb`z%_9m0W`(>t zI6>2acxMxV9S5Q(Bh!F%<2clNUnz3q%gElCav^WxFTwdO^8fmukA4w&zWXSF*-5xk zvEbf^XMXJ$>fhgM18m3QN21G^x7LkY%vx0nAb!>ayoEh8lZuuMr1s{APg9iyHE#RT zZ=$6{QGxI@Rq<1wxrvE8fs?!Ya^DBe4()y3eN39K&BEdp7~lqG(HB(bi}M z4WGO3?;Zp0ud{#EmYK?}KbD=8%hU-EaCNU!I;GM0;adNycegCYr9fh31_sksV8l}h zcr$_~rg(G==?Y!M@Utr6ahoNrdcFOZsyVEfuBk%R3EgvOiQ5WOft-MBU|L z8nD=!FC>^?Kz!i*Z{!wVLctNMlN;FuzFxP&x+3rk`j_zn;Wfk--m@U6iLm)vb8?)=+V!RZK6@ltQHkWX{% zJg0N!RYdO^(w=uStO6hdZT z&AKnA1iZdqW$GnNH-`*fEFe2kV-s%2*C=rF@~<5KJ;<;D zqN#CQr=o~;4-=K;86W3tDzKBG2P`UGl6u;(ef`j-)024Q7OyXHMLJM(`k`Z1k^+24 z)O&aTWORV*Uc$7e#vgEzjU#n65Jd85AVpJgp@X)Nd;yHJ6cjW zWLK^CR>Sb#P6l*7cDL*I_ssd7cL#OsYV^U23qLZY1xLCwJCDx|HaM2M&-_N>EM%%HK8N2lsV?7S z4pD+*pGKm;QIcqNcs<6|EoUhvuf=m*af(Ycg4Wj!iEiT0p7;>smyz@-WLGt=0jVdp zt1et!jGY!Z{h=N=&LF+@o%7v2GmorQ{`659Z?Mt-L?u+4$f?%Qcanx#3oL4(SZJ6RpaPQste zeqph2wBOg-t7~%{(FAKm+{bH~Mmz$m^J+PZ6forxxL-$Jt5;zAE3G)}v;5X8Mk&p| ztz4BWy4K@;oK)fGu^HuArEXDG9$(F};5ke{BkPEKv|tSjV<;xue=iL%>*Z^!JbCgY z9>Z)Q2U{o+02FWo=#{Xo-a{3TJ`~kTFanpq;I=%?z6xQIk+#ZTWrOd>W)phctZmst za4bY&&~o0rifGo|0UcUy*>u|@5WkpuU^DhPksPW{s$WK5ohlIgK|&H-;od*n0w$F% zo5I#z;gv%xKUNgrA<%+q$MXF(?}UWvOuaMM(!K>dztAT*>d&K&3{?u%QDCawkNU!) zw%sd8qyy_OJ59&#_JKjV0qJu^1MynFlL%nu?Ytd9%3UZJF_LTidCQ*Jx3S*WPyKV2 z>(_X1Jl+M4ucq2q=`!%iHkO;nO?YY8{qe+FCTMxbrc%6p;H|jNyHxx8!iOn$8vB^knzPK` z7(gGkjJLzcLI&e0@!JzxU6k`yCa~K_wlO&SZZn7d2}R!gGJSr(RZ28aBpQ)ih0PJX zQyr^sK||L8ponTTd(#>R8H;2Psh}WJltTr;l8^2sH%U(by{rNJj*8&(Y2(2T#KmLR zUY@wB>)zA!BtqywZ~2FERyU&6Kc?{KQcr2Y>`ukv#fiRKLHl=3U(?p)d=}qaFZ|#2 z&MB1$!#VTO|BJD=0Lp6L{)Z_g6cB0YP$X1Pq)VhM1cR2A?(Rkqkg`xxK|#7hIuwvb z8l>SNrMv!XgL*vo-rqa#4C8nP^gQnUjkP|t5}4Ad=`+f%*v7i8Y6VU`p_xL8Kx+iw#|AVkru* zR^lzRG{2GX|0$<^622~$Sl*M8Wgy2zfH=AL61cIQ6Z6Td9Ifg}a_>sqajoyS#1fkH z)!_XUTbXPen#c;warR>6A)XR5>r5tv^Yi87%sv6WP_W`On_w4scHD);eZ#fzjnZza^Fu|k6@OaYp?)YbnV+^NKU>7>b^CZRd^ zm9IA3u8Roc9|_ZApP7Pmn-r{-wxvS;_0Nis7i|0^JJ_{sa z^#aGZ1cEGBdH7&BRVle#2?aO0*uj36z6*`tNeaz*^13CfY2RWncPxT`QfICw3mk%6 zGhJzcVDO}XP^iZ21!#gmE9Qh)&k;_q?Y-m)A#iy(nw>*du74cmmE%(#xS&5U=zo>f z16f;b3WtsFm(6A*i`JY)(Lln^Yz%&tSxUP|Y4Wjgh@FV9rQKOFY zxe!ov((Q@h4~RoMr=knra}*UtK9;WU)}#CPer9z8$ z?5Jm?=;2~O0^=bjR(KwQG zed&L#tlC+x56~R5*$SBRt&8=JqJxVX7S}X~ zXwav2HGlGoNI!q^f@DlSi05w%D=hG{=HR<}JGX8G^?5`r?U(kr0xiL$$K&RDzPPW) zbw3AQjFR2U7CZA#xkM6}M*a(cBUU)WbjYZV`8i9NPZ(4d$bsWy7se7 z@O?93_~KZ{2Z8^=pjt!+3pG;dWAT|oA5`Z!20?`pjylam`fwrh&_PZ7A$AsU)n1q*2tuwi{8I=%75~xBND5<5O$CEF0vIS-h1Dk-FQ_>i&4{}7^ zuwY&l69WQYNZlZUi^Q>GrSL;yq7FQbByg23i5=c)&WALrG!N&8?~&$u(Ni0G{X8iI z(&sf-v05%^h_s+qZ8Wm6{iDNgs(Z`thrW`xwJ)Kx`u%36JwfxgWxtF`!8%}hG znNR**XKzed&S=y8Fa1g`L2U_FH2n?qMk>F6_xz1imnOu8L|49Yyg>Y;Qb%Wt1m?$3 zO*Y*0d>dI8&dWaDd_k<|kRZnuowAW9R7m2=^V(;axeq?wQ(k{uN=SCY9jtVmx^<^= zjXLoF2Brt2>eI|=1d1FD|55DedjcdWS+t_Nn*7%=-f2r3uMsfuQ}~X!tPj3=^x`-T zjSIE#+XCyw%MS4vG+Zkvzh+kEUz^y>Yo&_L)h7rYGqmPKjeHDmjDW3?*vePazTC>| zgu*^ZNnv&!j_u0IzE96>iCbQfzGOphiu7}pWMaCZ?y@AgviK1!UdrMnVErRo8D59m z`%TMG)9N8my4dpBwl9;6qEU_MeA|ScVUe$NH1343n$fY=M<|~glQP1L2&-NZ$sjLhKSZe0-#mA8McK%2(s1XHLc-n=w zl{~$sGabnf*_reBPan$zohDBAHI$@TJQpEtE?#(lGr`Av?KVr6RKv=*l=(ShyDOf{ z6MN))=gO9gFF)3!xE}TBgSV@Ej$U&x_;(pqFePeY{=%KFTgQ$V7HgWW_1KTp@UPxQ zyqdTC*D7$f*yB8j{#L-&_{A!9>bsw z!(s8-aNs2llysJMg(Nw8i16{o7}F2=+xyNy?s;5%QF>AnoV=)n8x zi<{CViu}Gs!?7u~pKO>({u_P1gpgFujmM67zhMLQz?Wa&5s-}nlb-VzpJb6oAUNu2 z%M!)8?a{xpf$g-0AuE7;UbQQ^f35LKMZu8DRd$LCR$oZFeD>~&q8HntU(v_7{zNI5 zgzN$#i2!+5x@2Ffau3v*lf4rB65qB)&e51z)Hu8w3KX>cUka)}~@uE_#>eMws@i&2M<)1a3QyyIV;Sef{Bs4++Ilvb(*m#fJEQU35%8}#L#{xw!L2xberT7tHf zI;pttHf!dx7)W25R-klaTy|x>STs$jPz$WAmxwH|EpR*<#l$~_x7lxdMC2RWW)3^j zbIw1k4JHEM-=!R6bvWJsrEB>ZEZ0m~+u&jm*JG+zpmqMnx6oVPSoj~l2Fv|()`0uJ zcFq6Sk~DJ%3Z%4;9~-01x4AnTxVO#FgRCpT}=|=KF8-DCH#yx#5V%rpKI~(Kb2tyvp8R(EQOk|u^}kcuHP5( zj+b!+qiI>4R)T4T&*U%MxOJ-<{QLm=8cAGTWhD`S_<>Wal0l*+keLw=N3H+g3|0E~ zER#1c+l|(Y5UI_gRzvT|$tQWwK8GB#REVSw^9hXVZ?|$dhD4&%_A@!3xS#`H^PfQ|G%ikj9;G|h7 zL<=X#hN}tFx?Q)?E|pYl_Koc%B+!3?qbb(rbU51s%}O?jK%Z%HI4?%F|C6C>Cvq3? zI?L)^-y~6&(r^MkHe&#k4>mtfa;TpiZ@>5w}#t-rLu3PO?C1- z1|#Udkx<)~BOW*_0dxUOMgn9UXw4%1R&FoYu^VHe(^i;%_9KE7g8#K=5oI=$(m0wW z|H;;O_UwJ00?6M*qw;}n?R-kQEQ3gxa~MP(iWWhuXE}5--TFB zB<4WA*IF)WF`HJ=7B5y{rob%o>Pu~r}uK)eHP75wuX_V{T;E_7sI-VFyK3| z#6JAT5(5o+Y_R5yZ~kk{>KD^4UV=~J&$-g8Uy;|1v9pezXM}z~5hF3?3_Nasid+z$ zBku!F8(FWs&X*Vs6A4flfkFJJBurp>#SLr0=MHwanb6t`(yePK?7*0V#p^NAbX-HU z@DOp6{s&_K?5rqF<$DsPi+<<=P=4|L>(y}x5KyCm2zPVw`mAlNNX3auXj?1N3IG`( z#O{9+yobw2@+cj<=J;$kX0#wC)T3L==<(8+z!0duM7F`<6ITm3in+27Jwb*BLI~9Y z3}8Ef>IAm$pXUBl)t98>{tY*Pass0Lh#HYO62#-Y2`(7QpK)p3vfQ4o2oZhKu|;G( zf2i%Z4E!%GvW)cZG|sDpAf>yb6YVq0$fv*w*Qxcm=IueQk*jV$d%R@lqFk;)?n_*T z(MAQp&W_fFAT9`qO*igfgvP`KBW$fcvedu`&<^?xq8GuiRMzK11Vx_^96Kj_GH`|I zT;Jg_avI~m1c=~1_)f%;j6P~b5oZlj@gt?kuZ6Rm`o9*A*8kVSkw40BWxT6$CiHE< z==YCr^jF@xtW#PHIs(?AcW1cYyn1H$&jXj(eSehiC-JKOxhIr}*;+ud_a`gl;YHq! zsCax%tl?FKG#&cm1}VjqKYvP_jP#nlBZGup3pAwixd!sgXV2a^((wi%B}aKPA=#Cd z=H0?E0?lz5diQ;QnS01RRTC5JaKY>6e-XRL3R%Z8Old$LU5=zj&KEPyx9lJD`Z6mI zL_XX^@6ShPWeo$^$DeDo^LM5_PFh;}8TyMX7cSr7ft0U4Jm1tO6 zf>iMW4I3g5Aaj~TTo~VU&JRB9T8VC){-^;fW>iMHgzTFAe+sS-w>n}}Aw@rpIo^N@ zHVZ$pX#DEV!^8vtbpei%LikKiNpGJ*(kY8-e85X8SQC}{n*~l#a8~5d3owaJaX!sl zVPe=mOZfl$K_RPN$v_YGu>2)HCPXZr^#O4dFnQYSZnMLgES6_fQBA1YOxU&L08ety zY4{m$u%IYTG_`RzqiGk1iOah3O3+o~_mjK+{cb=!`|JF#mJQkF{(^r~Svb4WgKsdM#5bCr$pb!T+iH#UytF?rvmLO2wOKzILA>hyF zd%P7Di)q>i=-)cV zrht@XG^Q?uF?=mr0=9SrF!U-Q$=V>+8iI^Otz#jImKdz9lp;tB!~y8@z0-pq#zXi2 zb*qy4G3at@05n?yBtW|^ltEqqsYMxOdkP30wjM*Tg@~og_Rq-39#1cfbFZae`yrwsApk;mK-u z8JzS%bfUDtZSf#+w1NhpZnk`j;5vw&!R=trzVYqV6Qpbp#Nwr%ErK3=43eb2AOj!p z3Xn6Nk(~3rcW_9-AjbY)cb`Hwb88MuYa*dsyi_hJ-q9TZKSTlYxX-pa$81{Wsd!cT zS?&a=Y|oU0zo@kOc~KACI?(mgxNpg(v16 z8yqHKkv#|ug&K8b4-rrTGK)ZY&3t1)97EQdi&`rd3a>L1(#{|;#>$UkI)6TY{`}71 z75$v8{$^8`mLWT-WWEu*(Z&=#_uvIb`-eao_f8HBb&Z-k$MuyK^w;W)M9wu|4YmGe z<}8b<8Cwx9%cncyf_At*ye+-Ui6Xfzy2V7IbBPzF8$wjfdGCC!x><95SB+^0MI%BZ zcU1YMU5M(&aqo8&+9`H5Jh^sA4{HckxjtL8mtAMN)Ez9h{yYPiBOvJJ%hJy(H$UCs zQ1Tdmt&Rb7O!!`8h6Www-ZsN0XZtv%ETIixIkXOj(-K0S!G# zhaT9_lbHiC!iLAuLUgJtf)(a}hsFzegNdzWL!hHiY&F_NMgYCJyQ;N6aENfb$MzoAEX7WlB=VAwz+os*&y$r z*w?8xnQZe1c%&HV<+M{nbki)+B<*fi>VhSTA0?OF&FF!jTz$Tg$JE|!rFx>kGHTg9 zP*^rT3y(vP>U{Wt_59K`8fv;Wm1C0)#{)X@28yElAy6d41KFF@4Q>Y zFUz-GPfImI;p}N*KNuvSi1M|Xn~;jnSRzU{mayw*a(sp}4qUA@9x9KipwIQ$GyhfM z6$f?4$FGE9ZcyUROi+9#_y`eAu18dGo<5~AEr~^&v%Eq z2?*rYEw0DkNbWCqzY~`|oU!4#1F!2!fhB1lEXUS_8^kMbTT&^j@bUd-uHj^FN$|KmAw_7OKjjFI$fog+-DoZv5u+kj95!2Nfl6s=jR_D=lBzWhrEhMhVUc+^2$&e6U6*|Ot&oN@P}3rn?({hTHabH$6$-tnbI zpD)cjt4ro{KIqQ~f199BVWGD(IY-It^3xM@oce)JtIoV$ZJfGh4D?OQR`G3*_PHCJ9r*PlNj@=bTwVfd}F z)dnSiXc+JEe|6`U&F+EiLS#>To#o%2GSi&6IozqpUqmq1_t-Q(erzV?LM?WV7}m10 z);Kt+Cihjk58`&!)>pVO;&1; z+wI#+Oj-V8g^S|DpJzRae_}i##2$1d-yA)mxumT*CN*3~)L@ux^fK>r{bNTVVmB8A z((+1Q=jSH*yV})@J|9OWoe-^13m)fC#wgj+)7<$^&#FP2Bv;AC_f_AWknAiW zXSB{a?gtcoH2RK0zwOdfl7<{;O?!+K_?IMPwnnho`^MZ4FbU}L^Ix(ZTb)8*{fXc4 z$y-9|5b$6mAt01`)EI3XFhe=!2%Z+vgHA;SZfQO(8Q$x~t`nut?$O@;RrlhsNAeuY zr(5t-Ix&*)T!`+&EkAIOI*turLy)`olcM_$E<4Kl%6Y;{WAPRv25dI?2yPURG5mPM10wO zmV}n*K+{{V-wzljG89P$CP}*0%x@mzI7DOr5`SbvDF$8yK5p`lOCvr++k>)1A`!3n zG7>&882(gyq4JsJ#=$P;RLpGwt1BX1saXMULntxl1`JKv<}eumg{<2s^^7a<{lkP5;1+l~)DY+e5j`X$ewQd>J8Rf;DvX5eIFVL7Sytt=BOq z3S)LSN!|t%h)k{Wtv#UU=eJ;bB3T>%!s#379!UBT*b-}L2JJtb&#`LuG@NcOUGw}G zZtAFvi5}!eG04{9C=Ty|_JF%IhA^L9nEMUHX(-Osw>vjmQN37Apf_5{ejF+Vl zF_N}le3N2_Ve7Ez^;Nv#RgdZiJhrZs#8&O!+leKTSJgM}`*+WuKRgQt#rp<|DlgsC zVS9h2C6`vV(BEupIW;)!C3}&N@Q%*%pv|6BoCI!4jBXfX*T!P+RzW#RY@PK%ozV&S zlwo!?{$re^Z_2q%oA;;N&uTY^I56rGgu)D!F^HlDr<&?B`?`7%WKkq%Ai%Phs<4(h z)#Vy%=dI=1c)_VfgC0&@-)gw4kuF$VVlxIz;}=-!SQF9U&>e3eACxo%_jO{d89ci& zw474|&gRXvnl?}EOY7w-cly1KMryuM31g>rZ z5QJQDER5o>+$^A`cl4WAggz%}QW^Yp@Z!J2its++-WPbLLF!z(TO(WhGW4oCD6 z9IgHl27d<)gw->Jpjl6%eF;E^5LHPri4@@}8!+KjtkVc~S@pEyRtdaV&skX*FpJ0h zYGGzSB#mpU|IYA`wri1kFd@$wTlQ^#J3 z`@@4+t{}dPMkpG!;>O`)-*h!h4k&9zpBiH#8{`@Jcw8z&yI54S?OcV$j{k~^@t#pv z?!C)9Zv!T_s-!hZQ2i<186bm9fb{f`sjL%ZL1uJ^M?nlF|FMs zwALiV*AX@fzWj^+MT?ws{f^J8RuITGRx4)x@~K)S8Vy12DXbafQ%GYV$GiWuW-_~gP_kmd$scrXXyBGRX}OMz zk8{SlDUvCi`2hm89==o4zuqZsIzWZSUhcI7!>KtQsMwHT0(YBvyxVWF5F1f60j?Qp zr_HWrv8}k^p#iwoGt;QS6GLyz({Ro8i#l+w#rT`@@Y_ zesGDX^<7F<`rvGZ^_o0g>y1vchW*#!)rTy&=5d$*3{1>QC&iJdO@I#4NIcu*I*B ztoM7y2!p5r{j;rLS@{WtBO@Ymyc2Kyz;@Z7k;_haxZ$10J8+yOOYq+&p{uM+j^;*p za|wY_6MRlQUvhVHM{TL)>6U&BIvi!^=C_8a7ltbrhfj2R*0EyX zB#jKma@8MR;nY2v6h&2jnIX4+yy~;JEp7MK6fE<<%Qkc+-HH_?KiJ=5xyfzdp(}9f zy{=fzqu(o_(B$tGkgNlx=VWABIX`RcopWz}3_`@9D7u%TAR{u(%$y~H6AkxRU&ptS zOc(ye0qa-ZPs4a#YY=p#oj7?0%VvRqKC9M2@Bv_g(er~-*ZIn;D%Dw49& z{xVii|4m#m%1o&p%&@aJb*@g=epl*DH{};L7=Fb@YWfm(|7gW8LZ#+>l$q?xui;li#^*cw3B+(y|R(iNB**nORizO zjh1|V=kFUOE*m6LQL)Gfjr)FL1qlyi}+szRf2DtJ$m^bZtnl{U69l-9|}p)1Su zDwx1&UAcNJqZ~OH1#A}SUQw~pXh|UaG<->L{G|a1#`+F30xGD?m53WtN$Z{K4W{Za zE#Ji($j5N z-Gm>V$NIc?z&4zJGCpVM0fNqHX?=mD+Q86D+%Jl-UHq@Mw|jIvAVC>XpTTJMGUBNX zR+gtZN&Gk|yXgjta*C8$A^Ld*j?wmCCA4^9#N!4e(_mk39R2&rr&FPq=qx%PzN%)r z+Q}Btd@Km=9APED=XuIgBz4tZQi@sj8Ki4fHi~wx?f38IMKN&T(9NRA6MfPYBSL1g zYH6#hPhx`NB24LajERDH!fzd6(>EM5mi_|Ok$1K;Xf`jtZ(Y0Y=?g?0-A51GlY9Z4Dabi z(eIO4I^8eMpV%y1S7D!j?N0&n0oW`Z+M6NX9|ugB3=g2ME%CnyS+$8XywCovn$9M_ zbiHrsg*p+vR;_e|$y#V(3ZzzzZIptJ@yRIrX|nCa^@Snx!~6({LdakcjIX>g-42SK z!VhO)JX#b~l5;73RY=wdvVb`6fnUSzjxwGJZ8a%`D{EO#1r536hH@&pa)Jf@w~qPC3N|3QO+ti%pF?z;s!} zApTCw6QdBs(5*$*`Xzho{SFYy7#be#1kZjL{n+lUYQCa&ES zDSh+JuS(ZnY0{TG)%lJTVdh1NPxoMNLFTMv7@aFtt_kHH_n=A*<#()?CB{HsIl#mV z5WC$;eN;>a6P%BMiOCm)P@}*Z@b<&7?!neicf=wz(jXm9zA5M2Azzrtqr~d?R?|_r+A3cU;etBn`kdwfc z1q#oH2QuwRE?YR~8I5QNO_R~6Gw*KVa9tz4mhq@K-_j0IpF(;Um8{@A_khj!I}}&+ zPKIj>ku+WKJ>?6@Yt($2X;qXW)IG)BeRk6yx@MotVjSi?pz)aS?Ngq?=T<7Vd-m1@ zj-91zz{Qdyug2T7-{VR%8jl%Pdf6EYQT0z5qqxmfndv7iW$*U8HX6jrG}0AH`r$;T z(Z1Nx1H~)ftZ2(DYhsia;+lJ*nDuklf*1*DD0Rv^64g!ne?C6>Rky5jd(G32Ham)L zR}|u67JK#KDr@CaDkhuOaDR5;Qi*2FowBa;l=x;-|Hm_u!QLnyIo_~kIrO^$cc_Lo~nB2p+ zM#TOX2Ei0;Becc5IPqEm-9%QL{1PnqqDxoC{LF?a_J_I{L$wx+4t*ho+u*;#^H z50zfbj46tk8$*O3x+}f-0H<*FBE)c^f7t(ZoZ%4R0@7zB`?K%yV>*Y)HwNsd#cL?9 zB~}PHt)55Z(W%Ng=ePslfni}^Cq~>BecUT1>ivL(@e7>c&Wxht(6;@4ry_t9h75iC zQ-LsA4Y=}xa82%Wu5E})oLDX?2#$h6Nv`P=LE76ZB>KmneBd5V*bhV_1ZkaUH=>eW z9PX03;6ve#2R>D>Ib825lGUa*R&#@3DE|u!#Kq9)8)#z&Uj2)+oG%85T^@n&rX|`i`!l+bz`S zD8IWj^1qW%ZJ+_Vm!U3vl16|8g5jzm&R`K>dM6;8_(iEnC1I;50cPs zM^v1hY*RSz4I=ePwwFExs)3 z78k&Z5FPLXa?CON`Af;3{BM zEHwG?wzIx+acDs9Gq+sUdz8+B+wfYq^?qv5t=~m(o7#DqLeuVagQU~q%oH~$IZ(9B z;KC4t#Ar^4TU>DhS8@{EgS`qcX$F~x=YQ2J?IImpk8oqoD@cH8e|iX_oUl#!h>@mp z%s%M7WRAd97vXQOvGGV2=Y2jwx5T$(tE}(LNkh)G&KQBgK2rPS2wSnK@kzVbuW{JP zFo|%>_yyYZY$Wa9yE;qz@bIPRruZ@69)rBuT1e~g>gad?CZ~(m&Y&s<=TgYm1k9Q{ z$Sk~LE_WOby&!@%SZ@^qWW!flT4p*l2fu57@%k?29hN6m`1#4fizwMsJFy*gVe+32 z!di6>yCw$sY2PuRfsZL`s57Gt569Klm2}Ci8rsl z1cEi>pbSPZo)QJHZwlC9S=;@v$062u8QL+ewScaMg{;L zUT$8aIx(#*$@f`({C#mkA)k~A0>BzvOa+Apon|j^8@4l(XuyyD)EN{Sj7&BZ+`~X~7{#vBBvU5KOwqYiLk!Ap=Lw@u#vD00 zoUm5d+{+#J?~Zt4(?SO}WVY-DYHd7lu6}`X(wimL<+5V!TZoiFCh9$aEE&Fg5$X-~ zcX)s3<_pp_e`eAJ%@z{vd0YN4Cm3kUo&s|^ZBnwgmY9fU30iw z!!tzG?&0<)pV{Zsf1gSJ>u!*ER6Ab(ant^0Bu|pn#*fKf5=_*(gh7aMj)&AUj@rAG zUO$!3Pl=6n9L6>ed}5*qzpT_OjFs6{3&(_Mz`O6g7L#xEM-jafB;~zL8FK*)PbpIq z9pQhGVS$Ky8a}@;HN9%S30E^|Bvf$kUzBd25d#o13sQPYpoHK#%#9bt?p3R83&M=Ag0AJyKje|J7$tuMev63Q=Dg&r>m zns&Q|!RWJ>YbGI+rs4)mv=1`;szVL25*Ch!--09gII;3+9J3-^?TyS=H^KcFuX6XW zSZVY{FPuE1xotDM79XPG+f-WCfIt5y5em-j5k_^Izk`ZSJ*2Tg<7}_yZ(B&=8Gn0~ z=n*q{*!od@96feiX{EQ}W)$B*;h>rV544dGP!&niNK0i-VrQO zrWbqTvwcLM-oZa;{@F+L2~nMFLU_YO{`#Rpf@J~pA@E;hW;*2-NFqUa_cF-9WcFL~ zYO!rCTqKrNehRT@@W>^kg8*%SED4{a*=<=w&n)bBl8D^eb1D;es>6f<=_!?%w*e&X z&4g8BX&UC#=+ic6HG%U7Yu$B)`%cWJ{ZE3lzl6f95xo&jvVQQLWPQ~BGL~OH=6sY+ zu;3`0z4!gP(T-n@V3tYmWu)nX(5A0XSz-+G@c}Rc`Cm1{dE5u~_?_39NW>Ws2&tT# zCo>L#(ML3NCUQn`)-|};H^EKVqEYCRc$LY z(0Usc>mC&Rl3nq=!_w%Hbc`+`t$N=R5=}sLCAdb_KLG<4j=rPQ`xydcAabO`Xye3z zqo2{-e544CuOWmEJd2-b(5a2!N^+R~&{O1E++q!yl9XCRX9jP-_jol}i5fQ-MG_(L zz;!;3z-hYrP)pb>GO}2wi%5m2vHB>rSPwC&F#{W!*X(!b;+lLsh$KgN1Cxk`FG;J{^||UsG6Y^ zVG$T+N79B!#_-5520^H}V^E09BlzZcR7L%XBT?AvdAejbEARQ1g~wE_>DIybd-ucg z_HH#s`rWLfl)9Gs`180@!CnZ>^`f?T`N_5JH|odz&Tv5=W0c0Rk z$eu$b>JGt)JnYYSjb;{tx#v_Mj$_QSnvIjdjyDw}o&bJb3Z9lOg<7N(w(v`Y zu!wj*-dwbOsO8SQjDQmfrGD(Q)eupsd!5O>Mku}sQ7;2&kHzh&Cpf1-!1OkyV%cdP z=dtM4XFCaYjG7n@e>*IH*ZE9Ex(VdX9|@KI*@k0$w($RV=aFMRK=qAPx4J3 z>Y(p{oNiGk5=PVz1j#3`iqOhVU3k9{;O#|l$|-GX-d;Q7(=k-_j>|xX)!5+eswHMs){+ zDAF2!(q438y^kmzw33?To7=Fa-M-ZyUmirXVUWP&EqV#{qI-VdB+g*B#dcpE0}_@x z@9uMB2mIE<`X$E>Xzrj^K@jU6kxmrle1;1nZmc@^8=IVmmWzmJwXvylgAxo)3f*Vn=k+m@5#s4>==(UXPll%88J3T7IIEM)W9%&!Uxu}ghv!Y36A)Z z-xeld$MD$dfj-Q z+8q=@!HrXhqK$K~EPSg|o4PX!Zpdx~K3FAM8I(|cbBeJU>HBG;c51?|7eJyz1h-+h z<>kf7a_ANej6CHM_UfLx?ySG@KP`f@@?^<;Qe?~-P|>f@GUzXdyY8)Ze{2<~wzycf zshxuu+MngbL;5lV-O(XqiV^vtqK2z&MKdAqwSx|TR)nkEkh+*DNFc&E`v!5t)-@fQ zQHf!gJUX#!aozcMyzwDN+w>1dtJPWQvbyo37HjhrrcZN7PRnt~kSN;;mNfQkxKt2D zbs%Z0doM3JuknQr9KMGRnEzTq=cp`N+c2eriD{KBh5WRMDwF? zZYSE-(kMI%{d8YAUi$qHS`8O;ZX@04$1lG5(Y?*p0H{A(K->pu-oVnMiF*S-l;3)i z{y{EW*s2-rkUjHejO*s>i_c)pwcqXMrx?*GAQ0n6Ki=#FTg+XN`p-tvznc*4Oly{k zs=-TTbv9?a@E8&Hq1;Dtab*m*cp2Q7OfO)qP)*hgMU?lN_-`7=Zqr9;rwNX!? zd_lq!?H`hUw=c$vz-BYMr}=cTQhX7r<*SF?ACWdq z>aC8fF-sISe&4~~B?FWnRo_1&Goa-(`KRT?xrt3f$Fbb>KryueTzV&5@4msT@2lsTI1X&JWzc0> zXvu(^kyivcB{0nnU9^SN$EPgfBn9Ol7J$jjCbP#ek_RH;6|z?7+%jXB(wG|muVVv! zT9a+Omac%B4sc(75nvSp>^3LJhnE|!pI#%%UN79#9!AA_g2|9|rwN5z%BzQCUSFD(m0YT@;FEo^G_!SasCHKBLRZei1OHzGR}~Jy z$GVvxR6Qm9oPMpJZN&n~Dz3CBWq{QmhO8t#5?n@s?nmn!v*PBLK=OFUmC4D48_&-{ zF^P8$&qp?loRWuS`S!hg;jF*(5Ov-JWP;}HSP>#W*zB4!^`7wi$*25Z8>`XqFbZ3< z`gqH{3tT=Gn1o951kg?wRpF)u%M^97p7J_W^9wsF>s0Y0h2aD5(ItXub_aKar(A^0C=A+R7}M#xAD z1j(Rv94AWED5phY3ZF^^UE{!%EClplYqY%}`MGyiA+G%9Q zdVhi0k7O!;`Wtue)_%Mbd8Fdq94UW!;oJc$zKY0ecI;d;nl5crx+&r7U-|5ZMB49R zQ(PZBpC9z{i{i|RDWs6eqogJJy-%Q(?=;_BYjhrqilR$psCpYgWaCz-6$GeUApk6* z=i(=m-WpuZCryubNpqYuHoIH1jzbDO?@7wbSWqGG!sPL%*3JCbcOuM=b4s?hfLH5} zG(!9oB8?!571;F^xRSwssr62zR7p<})%KS^V~GAfQ*Rq?PQ0#I*U9&AFeCh!R5ZiJ zPO2a~*N^RTAbi^N@FOWhM9GhHcGXXWV^&lT!@T21u<9*tlowW^x2LJ{A_hPp+vA_2 zWfegr_LA2l8zlhRh-QQ|(K71xHbUA6%s$M zM^B@TxEkHdvyBUDd690#{&>$TCB?Z%`d#8IhNpnj=`Re-5+`8_ zz+f2Npls-xb64b^#0zSALAyE5Hd+YptA(ND+E(6S5L=}Q6L8*b2Kbz1WyOVHm!r#S zY$4-Z|2zMv1)y*;0*H?5ZA~x=^TGe2Jg@t^L5NgV$QIz~F zv4B&uLQ@}^yd6%_(TjhGeImxss@!4rHTc?buAR=HGB0T(5pthRD`7;M? z*>*p&*MhEGCKt@u{+v~$BH(ogH5i9R%*wtRLr3;w<`>Ax$avh;ws#*)@AnAkjt{yc zM9kQRn=c-E!2pIurRjWM?ZyXhAJ*__WemQyyC~`IbeZ#WDsO7|=k@Pu%f5Z#V9Mwd zc;iu|5nc?cChDhkYqs8KlleSuy8?7RIDyfC0Z~7k*0PfWB4QU52Zs*}QVan0IrJIh zW{mZ44SN%MfEpzz4_3OoO7G|sH_Lx!!u>TX%og8y9SwBp;M^B>R@rbndkce51ji!A zaMj>03VTF&v@cKQVB2^O=Gh#2S$W#%Iz>kKzt_uu2te62_e<7qjXdFEgEVjjnx+(9 z=}uxHRA)3Z18%2mlHU5Q8aLcmF4ySLD-bJof=n1kzx~gFr(SF#)FdZcwRWD+AKrm> zw$uVxK{O{$(wR|a`hbXb&qKgA!n}oSG(b+DcQud-K=)r*ynoMc(O8mUPYET(RXm3Y z(gs{{zNanXC>tM=OEG(>`}acI-XF_v|qHY2&K-ulp0oJBfOR_4~7FOz3)$z{ni zC2(F5f6>Q0gYbAG+BB^7tcBC7k&89EEH1O+$EEVA43W=n>5G{FCq8lyA#Tyg?#2^R zEv^1UOh^#Vh#iep{Ag7;ha-&90|JQBl ze^nth^XE?C;xoexnd&Dn4+pJV{RE_dnEGbn{)z3TcodN@e9gT$nC1LO50JyHRvw)jwG${Sig z%4g+@ADHeA%<6OW72@1dxOgr8^D|D=0>6XR{_`o}G~Txg#sNlWlOQv3L@ZNw_Aes&p zbHZhtHB%QChAb6Xxz)Hh*StY;4SuN-evbhRGX^Vwh%--q5Vc{p65)qjv&{lQq1~wlpMDT{Y4Th(?i!Rk^lD9c)UZYwl#H-^|6dg*sO}*}M6fR<%cn zh(HV~&(;{xE-0ieQ~#s}iALg?-#pS^4MsZ5|D)hUs|W7VtMu7b>R6Z4FsmRf-Bt%L;y1P51 z2Bf<|8fm2ae~;&!_kDlg`qtvCb&kh1!Sg)(-uv2D?6M{QC0~SU4*dA4&gymIe|B;7 z(m>{u_@mNMhiJ(4*gtCzba~GNE=M*Q9(3)h!FUxe9$7)cfy5f)Y6T1vCJW-f zaxS6Ecd$(R#R51=D)4==oaI2PPlDQZD{3 zr*(^#VtShlCwNtoIwK zsE^P$9@~wQD?s5x0yY;k@^>j%+p+o;HAaX=(9G>1f6SE_Drc)vkm0MJp7|u zpgppT664c};8dXYvN*JkcdR{%L$NmnR@+YZ)ln8B^nA#nu!WhbK5nU=I-aedu%b&o zMI^uT7Ji@m0p(@wkw&D*=7F^cFJtLql<;#Dmv0`>)5w6>`e$ha$XuK#w|xDikq`oZ=bZ#m*L6Y@sq3GVG$N-enmZ$mTz#rj z4Rllg|4z~f#cH2pkWfVNws>dzwu28o^?}p^eab&f0OJ?PvU^F~ePJpeU&O74)o(oc zT7=&Lha&tLG1IFC*EIP{+kF(m-hZ+H?_R? z$fa=UtgPx&b@nIFEOowzt<#zr3|PICA4Ds!2*fLPHe<3MJa-)`Ggri=BFP)O}k1w@pqLM(00&$JSUqE{ykrHr5>pK9FU^ z2L?PusDS`z2|)AO40WA(2h+uDR!R{6Xb;+iMZf~ULmk365`CIZts2W88w$)7fwhw1 z$=e&q{YH{kXVi8}W7IF+{l~W|LEi;7y=eL#cYtKUdY{sW9=q(;;rb7+HRQ?KqTsWZ z12u{-ePId2YjpnNIrI+}`wECPYc7L`C0%7JXum}h8#5So(%?1obW z^#@bMa#YkOO0Tvqr{^o%gWug_?KwN;N`R4~0gugA)Bi2rEmHq$*dlX#W1^14?Z*4p zO3XV^N=sF@d8F}LjQ-u_ZnOKg+TR57@LD+|K zz9u`D8BRJ5<5?*u2VJ0ZwHUTUIQjxGD&*|zoJ>)s2{UqF(bi=40EYfCcu^!$jnp4? zi|z=E=w|<)T!_Eip9oTv`_4b1$BVK^17DQ;({7_R2)zI3I9sPKRD~4D0nF79KvOTqnaJ=x`wcw2_-kZ+P0mLW*@h13B`-~ zH#_qG48sI#0W&M7Yw(qd+=X=Tsh;EEy>w67quS;jiCijKaTj0P@m>ft*8f0Hx zV?Xui9eh< zxBeR<>5w2Vi?jb9h3stuZ_`p?vEO^zIHz+Y{_NM{v1aLmF-#fS>5hoW9D@rj`DqchF2d) z0oV@O>KP%FpN#YF926DtP$5oAOZTA@OIiVVqwkIBL@&HBmj>e9a9XjmKVu@8_q1~g zP@G7U99?a&sf22R>KMBh#!U}ru>5B!xg6tk&@}MQL z-cVwy-2Jt!Ur;L_vGFIxf3|f!4p8jrrAh*0 zoh8MP*H_UI0>&SXcz@53fecPyBviJO6deF`;NkoIqO!TVWGEdt_nL(6$;*(9|4qoo znL8V_HRgTOJglqepJ6f-6-l>);qbEQ!xT#CgSz?M>84KJ0RO|yj*dl3NlK%iqsb9Q>GMn9sBw&}c0L!RTK%hcJ9DjYLC(tc|qD7F8Vd2;K>+@9Vlv`)Z z%8)t_C|xbyW5%cf=i~5WkhleOsEl(P2ztuO{XA04n@Bo|HO9o&z~=w z+lmRiBK*%S4c6Rw&-{gPsY6|uPu6l+5X5md zXMADfb)mGj^f$SW6}z2{+3WV`v~st2%z9!ePmE$;0fw0PK3u#_Gs$}_cR@wq6B#Q6 zW`R8-fBa9}f$d?8!S}kQVF#7cbhyF#)n0`+@F{iOO$eh%SqF(@vd%l-+(~$Lr$ewz zDVA$irL!5evgzk`|A=URw*AO18ulFF0UgLNBB3V-A0F%)EUkP_nM7Jz=&NR3D82&= zZIp58M>D{x#zia#CCyJRsb&;>mX~9NmK#4&GG@3e{Uv~ zUtSzU00yf0VvK>`up*wVd>epwfPUoqa`*tn30{`{h0J-44HPc+22tPy0Ee`7bnBImJLKez&-HX;-WQGj?Rh#fT^ zeOOpm1_ar6NGkM7yQ)fd^G(2r2afW88Sx+N(KfI35DW9TNFC9pk1)%zW$nmZd3&RR zd;80m487Lo?9OwV5po&|;xSly#a>9q9&O=!Wn(=Y3gN;g%C_%{(!ay)Xfb+_C z=MP%gzUr?tZviYeU(?7IXej};y!vox3Cpq?E{XGxmH)r^Na&VUohM5`FLi)xbp89Z z-!i?GAlq{(f;fy+2D2$Frx>e;^p`Sz!_=KoHZa0C+s49GHZ{-LFHieHEc~0Jr=a_0 zJDA}4%*4|oRl&-^E83gO%Narc4i(qt?%UrwwQa3jW+ShwwB(U1n|SSG3=tt)4NtNxt}u{Vp1*Mr790b5c$5nAu@0Dc?_Wv3|yNQ2@{N; zA$tfNEFOLDcm!;P+xf&UOPqhJsH!w}B?BKxi z>%Br~1^Srr*iqTumV5uVHE#5=Q)eHL5_V>q>$Ob97B~B~i$; z|2Apfwfe}-Y=N`-^nI|hl^bI@)ik+cukYPpo@X2i5Q71N*G{VPZOWvH#OG)-d@VPrHJYgazn}Uj0@s`VJ4{ zabsgM!!5R_EjGa}HoeE@TKKTncP$Z+Tg3{rqb1(sNyp>wdJ0}}C*noe`{4gS?cXGy zK`Q=P#GVPDb4myhDj)&0`1#;{z58T5t%J|9lnf2mlj&Ri72bW9-t*%pM!%yolm`i% zxW%5;EE5V{?zLKs7bbqfa%jm8lc93Qh;gi_!{2Z7Ju?et)w<1P z)~z;zY1c9}Zf!^Prn`~lvHX}uUK0la-R4O?GNHn$XX(cz-Z$}2nLeGqjAf`kp3zs~ zdDW*`p&|Q8?w8exIn|f^Zvdk5rY9-M+k>gpiD55+yC(9^Zo8M1Q?WGmYwZ3`EVq6ObcnceLC>0www`>uoCmF@m_Mu1}Z)Q!;T|qG_k9FEt--XD)&PN7hLjd&_gW1?Msjvsv(9qBxjfNHu0qZ4+e!@Ra zuS^nH-N0cP*JT5NOssYu(0>Mw1t#7P06_HQ8-yyE`k?KY5_UENExr)aG zqbezyn7U3I+^avrj~@zZyvgiNHFxSw<_UYTnk!iq^CL`1M;LD@ziEIKD zajg(NP}=g`-}J#WYn&eCJ}B}hAo(ozG#p53*`df^BN~eBb~?VT-eo(SIq;>#WJ(%f zKH1#z@gAV&RiP9Eyu8XTCLqP2z2i=cx8$6W4)GHunq2p>bQr>`?B>!$ShmhZJu&RH z;^#p6qhRN1t$&@qQ#XIfw-X(FZ*DUs;)LSFj5hV~iI8hpQnEuQp=CaC=c^7fOM;cPNaGz5k;bj zV4rD6+N!98kK{gtjOTu&AH!TQuEwCE{x5Z#;0{#O<`S*n_oV)R0NB)280%M7ZK_>o z4{T_MYjgIl%)%Fsm}HQ9N-$0!=g;K7^JctIFLRwAPXp|&zA~xZEkOy&?#76vC1BaQ z(2lSCM2qLKY!5@Bu)jd)DTxsK&7D{0S54)yw*+|CZ78YD)dh%3TCQOi`=|9taM15j zoeAz)=!s7H)0r2l!O4Ott!WNR9l&wK$=}hZC!QBCM$d(eNu$uF{$-NapHIjO=VGui zZhN4?s$0#5N>zRMlxy4YKXt(SuTQlIg*^_NLCc08yyxW#`0QkQFsSm-e$&lLf}P7W z&!!oT|6li$yuYy_z*y{CXnQigo7SqfWpD_=Kf`Sf134X;eorA1CXC0*SRFz4qU zYW2B+t=Jye+%5srn`V%u9J0P{5%qR+xht}AJn#TzL5|WsG@h+etf-z%&tGHtrsw9~ z1!m$J)*A!LGM0?s|FXH4!p8@olQseYtNi2q`}Id$YFvgym@ z&wbWWVXm8JrqX-a6t_+`|C$8am~3$2V=?|X`}_1dQqs(S+)E917nh{xNg{up)vV>p+Oz z0ADJ*S;)6{fde?)0q*9#f&uGn-uLkhP*_a`pG3VpP}aO|AD0OsO*!@k2uN;Upk$D) z=XM3KG=%}DMT7w`x&y5CN{?PZAMt zYPr3mqB~v?D+YflJLLll0rYSWccR8`$z!m(P}PXJYXvi}$KS~>AMEJ3Q9tgB54>i) zQuA&WYjuW%Pj&Y8s_8Sxd1la_HT4Of=w5;#@tk#!Grv7*@4tA`GsXAv{_W!j3E#Hs z37H7R6VM2Eu1|-MC59Kbk5n~fV}IT-hlYhr;9`(-G+$h9^h1jfud3gUDe#LfyNk=- zf?4`(zQBI!yzvH#X9>Y06K!uhm5$^S#DYi~sSqx5a;#s{fcJ6IKi$WS`cr9|7R_`H zQ>wnWtYhaje-^!^>0eUTv52@MN<;Vb_PY-T{97Z9SRq6!6y zgh~CSkf9-!iU|@80w!kWQ94<9LWgwOi7@XSPK^uGB%w8{CKug{D}Qgz2%)!txGoLr6zf4e0um!!Xo zthlN;k)F=$_a52_rktL(EfQIKGcV#bcsw)u?1BsW{rhW)-AJM6O4xXceIAMP7cZ_{*!-K2?7m`+3R?jw^tugUOq_ihQ4um9Q;T?xKWZJ-`L z5OkHyBB)LseyCaLL^igqi+I~LJA%vp>2g3nZOyeX%jmTH>V4bs$)b1e^_6Yg$w#Bg zrg%%AJka)$a&{=!;oT%zTnO$IZC}J!irV^4U6)^3UX1r+l~&oU>}OueyK!wQS13Nd zRbe|cwsxcFF?GFba3<2{we;$;beH6n76+yxu1AXtQ22_ORbE~FL|c4`a)T5)o-pA) zwgaL&`3$A$919}GMjv1pq6XJbgKv&mP1d;<W$T zOSbI{f$!wkvP=)c>I4vrUiuAe*?gF~LRSOg`P^Vu>E$R5 zWAiKpKKBj%JSQeb2cay^KCxlT;qb#5r!iUlZBhpZzP)~5x`m(2%F61Q7v5jScjp?W zAJ{5>k6e^(wD3IQXK=S!z8MBGdHcn-I~lDyYa;^};KQa{Q!i{JEPKSJ%t!X~*0NUF zQAi)HI4U?M02Af-#S6JP*jk##-@n5x?X$MJvcg}E2ipz=`^QOVH%Ep04T@hpHpVXv z-jUMe2;JGw*oQDid$}il>)=w0fQ%+mkDfJ!-XNM>F9bcZj?~xE$8RDj`?DK$`z-B| zKP)r`A!-lRCK_$)H2QLAPvj^JxgShjqgh>C?1@aC^h?h%NZ9!D4qxu%k@*jAi25M)5z0lhXC(@BwrSktmT(?fkZ$;G#LpEjapL}gU7t%%Rquik(0 zl4kO{i{;DyDSy~f--_$vBz2P1N%jt@!x#=+fy1)1(fc%Qo7y;MhnR<8@hA?Mju+Pd zc~QZV2-nso2i70&U){+QZU-d&kz-&bk8)&R1pjmlrpeg$cemRCOBaxx_nQMVx=hP( z3Nt2DY&h+t2hIKIjk9s5s?4sIp`*NsiOJ~N#&(4InyPZjpL*i8Wf}otXbLcI;i5~; zJT^h%*IE!f1%{45AOm1HN|Dx~Jd&ra0Xa|b4u7YGDi>YQpGQBF8XPNlAtQTp&9pof z7>E8xa|+>=(rKM*J-#Voxh)}}vh76Sq42%$HW6x}pqg01u(sUg{Y6CP95R^ld)UN9 z`KVE%QC9N3uyoJuvR%QDZal z2mh(AF`t6;Miq+L7;9||%d8=h@{^6pnEp8~ipVv^j_=}isxR5M+2HG{x}z&hVN>5J zn~L-C5icQ27QdA#1>JGa9-Z+yE(T7yE-w?<2$d7e+ucyXOYFDAabH2Rr@g-%D}(xw z$67VdI?X{^vWq<2=-#gA#5+iR-nhx7(;zZ%e1bkbFI6|5M(wzEA((A*lR&9i3zrBC zo{V9l6JflgHeE?eFO*Q(^p1e0u(8SVV)!@<+#~b(l}&;?;os+Lv@h_XLn3c3R+0=l zmUkJ_EZ@A&uQP?JXs2W3Z~HDKQ0zB}UL(G)GM*k~8k8J2Yuj{9`yW<2>l{{zPF?#v zM5Cj6y#InlyS#JtR>Y5>k>l!>Xq}N`_qD{VYa`Tx;c#bGXIrPJJS#)zx$(NgUcD>UMAiQltsixf`GG_u!?v$x)FET(cM+&jM8uIuR=`fyc9@5^@4uJX0UQTT6W3E~ za)&3SdfnlPk)L}GU*T@C4bYy~HLj@wN9|)1!LZ%7_b;hb!;+d#-GRVlE#Y$X*!O5i zqr~{+?lM)k)Wz>@Wfgs6PyIYZ_UO$4!}Po}1Yd3(aIZ|Jql@FVwrq~VCK|mxUwz}c zgVIf%FKAKH;fY?d5fTyED$@ZaTssiZ+7HX~E`ZKi%-tO*}*@2_oaRarV!!v6a4;||roN-%FjZ(@eQmH9~ z2MqDmvuks|HPtbWZ%b8!ZJO{O>ohf}t+5jt&%C??`2IfKU6FnvWslmWUfkW=Yi&*o z6Cjq1k+U4mM1vKG2B5I?BcrhnEh}7_isJ{&*4G1XA1uCV_`MnbmB>)+=dH_vlX?7g zu3)QxrsH#FD&5QE_XiRaI>j2JawCs3&WJLd{POi1$ia@X)OH?lIjw|&uI2Ki>89QL zWH#$hz}2>$@9PLE5G8>IdfqG*T}v-uGW7Ge1CTi};SFXhr=bQ43P+JP33_Ln#kOyK zc%GoWI=N}NR{+72vxPI+p~7Lrdp?s6F`wkOnQw{;3hy~Qmr}!oB=HIVPLY!&k^zFZ z3GJ$i42QPgw+s3x*`1tLne_S+*g3|vuuQL}P7d~Z?$EZi$+jU+_kj(^msd07w(|{w z_Di3cSoW{CY}(At=xlz8ML}Z0@53_iK*C3z z&3eRWXx5n9t+x)iu%bd&x}oNzHO76EhU-e}QUmiO8*B{_PK&>z#zuDEr$0oe+`Aep zd*Lf<4Wi?}53~EqNQ8q+%tO^eL!=k77@FPgu`io;#NpPqHCIuGC%!lY&Vrf0-|k^c zT+po6G=llgZJqX4q~fivFjj)I#Y_B_7JLp1eDRFOvbqV&>3+%>E8gri|GnkSfU?G6 z6@h4ZaAt;JjPY=a{At6Y>kRk20OL*94@J5LH39#H#;5ZY^<(>SGx4*Sb)%cV=99(<#Fe$ zT+XFFNyoA`UJWao0%3J;W(f5{w#gSx{f3_O;rZS>gGkof<_#Y+w1HG7thcpY=ci3K zQO>y1%-{tH4c}~cwj`e@n%{D4M<)?Vwz;X!bg-b9B`%0Bmv*zLepcBOKFEYlVK==F zdWi3P`_$A+wOa5Sh@i^_odcQY~z+hh>fB%rKme))7C}%j|HQo z!WXbcOQAk%SE}>M&AjNjI}HR-vH}r7xJRy~isGk83A2Q_g~r&7#BXvk$;Q!}FQL!# zmjGKS-mX#v#A9alg(cTgjV!p5gLlhr6?I;IxLR{m>sxRdqrHX1P-$X)CeXL~KshW; zQTXk~=|p0ngGNsEhXfe^spCORt)d^tMk9T-*4n`Q2JH~qB6b-=dmTyx`)ioC{dlJ) z>v&xDN$i8U%`7{ro#aN=kp+_fz8bm6bZ20T@6jM_U(n|>3TOh90%DuZZV`wZtO|kj zPSOTTD)Zf2;W61@JbTLM7ONQ@81fZY!rX^!R-rc&ro`TTlo3iZx_`0TygoQ45W+iS z$l~_)1B5pPa`4~myoUz`NBrA{LJcq0uWC^XVuNT$RAjJD?dT)KSe@F)^eR08LXIfU zL~@#e;8B&gYy9gqL;wLdYe+3CcyM2vh?Tv1aX{?shwtzgy&EQ%Mx!qutaHg-j$ z!bOVrsC=GHD0bY3N12WkAKse-Qvp;pZ!xM z!^%Nbn@L$+IY4lAZb1dXwvcy+`+IO`pmb8Y2#-0kMs6*4Qg!Zya8y+M$E^gLH<{+f zhKP4Q=5L6^1LtX$o;mn*YUaHpF+8@R`RTpJ4PIP@WF>0FZ98V1?U8kmqim<9xjo{C zWn6Jsck?>DU=V-w(n9M+#BajqeV)C;t!YnOAlPd#Z zN+7uGeAD9Ah>-}?Y(NZ!bFB{3V)mQ1jMl#2966r+WWH4KQvY>ij&8=n(ILWxKZ*FW zc6P>V8-}ppl1;sXmi1%yp+YL-zH7>KWEO=s-r#HTr;CIz{i0`zDgQx$j%6X_!@O|l93LZgbZ@v?0+gym`=iY#IaMm&4b!3 zc6@%9|1?>VI!z5MiSOXkC+-N}+xfw;DWW`G^4Vm+N2=ejMT-cZ+|k27^3~XvE+axy z+EA<<)|7E@p(b;NUvK8SM+mZy+xcGNVF*=>lA#$p=H}+2l7&GS8=1hxV3t7Jp3fgS zAe7CY_Pr4V>nDXYLH|V$C2TlGAON#M4M%B}iQ^A=b)Zle2ws=#G(OeF3BE_$k(j0| zX^TW_)Pn7eW`RV@jK|j2mFU(vTB``TS*@1hyD5?E_ouC>2YzsW=%xA^qUYJwI;<*@ z8qHHU8%Dzg-a!emFOYuRQ64qC2hbU1UH$w7DDO!lnIf*w?E>+i z`#Zq``spHBg<6UnM9dRY*zj15|H^Q(gbLogQ`pgdFe}h`nub8|lK;1Iv8f~dZSSep z=Jg_Y^R^GB60?T%5~J*+3k5D4R;7|F@zRpfKae6u&Q1@K-YJS!2ngx>kX2u~{Bv;$ zt(N2UEV&&`v6SF2jSaJ9$}R_c=S~w1#{6`#(>Mon?L_FlIszf-!1dW?V^6?=k5qWN zGr#ORxT#J9qnJ@#R*~ra@pQSr=icJlmO$e@*WRXk1~!-Cwn!=DLQQZ|>jWoKAia)* zS-bb%^m6s#xnd}6^&KLd)2MzaKE#f z9<;U6>3|eLeAT2HH*;F!l@R=JH>o!xL~XFFC`*4a=Fp74qpkPMc=8oTCS=k#$8PX=EVDGat0^?+yhi zqF*ALGnyrIW-sL?{1b9>nE*LZui#1_d4ANKbo~bIQ*MWsuc>UeCL^YY2cGj=<6_Xz zoakH`3X7cR1&G#|SM`VpT{X z))4N~8HUAirx$$ipkfn)I()L#&UGW=9D{P~4({aBlK@`4lW=CFNTd0f)j`I69C{PE`yWAYwp!%vSgJvTcT4 z5}qkH^nDn@2xekq;WSU5`1aj0Af%C%o6cChnwsFXzE)vS#I%gF#b#+|t3JvIHQ7>x$qtiBJV-sTb^$^KV#-w+8NBy%Ph*7iB`efJ47xh9zzABEc^GGibQqRGLh)+C)hb1@ z8Z)XW1olU$zN{FH!)NH+@t@T!_C^cHz>{E8(%5%kY-3Rf_aC}#(>Jf%+C|$J7LeC1`2iRvPt?k9S` zx)=(==Dmoj*Xhz8f`@(|R!KRR`naeUwHAnse`K;TkYm-;;jGbS0IJt%^vx9>sVG%; zJ_hh5H|r}W7Gd}hPGflWL5wokns;ULC4aXkX| z7Px^fVxpsDLZiHY#UT}zYTNhuSxMGdK^+Yk80cE*Hb-a&c?fOkAS4bh6PppXMpMO> z0S|9bF}w9GEgzfbZ{B)e{(3*gr3wBdaQ5`6wG45os~L=wjtE^4xBTWf(^pAbdchz&JlAZbmv%d^(d zOQ@`EVi4G9c>j@T(mgO?2+P5VnMJzh9R(#a?+w<5&4@^Q4ZG<~F;TH@x&sfoxbjd8dsr`*b`;;IK8JZ1&e+VV?x6#9$TuJ&-Qf5RBwWfbFDoAA3ZFl7L35!SQNU(^p;$PhtzlL@rn1*GZ8+oK|M^pIeDh70h9#!$I#lFJF?Ew6SlmC`*( zId$xgfj63WRX-w)6~Ol(Sl|9CkPX9!gNodSx?GovE*;HTvZW>-pxuoeldB0zFx+xa7B$5u_pWb{G2m!i_VR z87#FDbi&fkRPXkIkp{hOAWKj(XiQBz1@5yJ!bVl%*Lg@ug^L3?lX;M`J0-dmUS%us z0dmi%-_ut-pJW+(;rPybA#gaH3mA6mOX`%`m$%+5~A zcv!N6#WI^Imuci;i#o3YdzwQ%A6Nn6Zm`Mnt|6q^s_@K}5GSCrxFJBIGUytjS*rpx ze9sx#j6V$K;?TBP+X&aI@e7M+UZ4R~uwq-3`TFU~VRa!A4wkB$hYgF%_-nH2Z;4Bt zN+aTSGz|+Ix!Tz2r0r}I^7%e(p-)!YI_O%2;`aV9+MNl9zfyhG;QY^)auG&f(wZ*U z%sYSUGr1N+Mn(o%K@h(9W#)QSRm9gHf}z{REZwfpibnToOwv%}+g#n+4_w(GL-E^~ z0%C!#jy(_xtjlXb8Y5W)-}{HVcBlh+KNg%GJ-VVqyJk(n59`z?j;pL@?wWXpw4auB zz&J&Q3rs#4u|R^;|IoilH#=<`a9JUBcRGr%KAVo;1cLLm+4m^W1`-B*VW2H6EUb={n0@B{0|K$E-HuM?(V>DHMXfY{?_yLB z$MznwjX3tkya=CoI=F(j%|RqPr1NPBwk8{D>nojy8@0I6FMJb{ZYJmBE}WhjKw4s! z+mIsFCMz;HFrXyynUS;n^?lS6NtqJ*tNHIUA&JaY0~4-JA~jH!&v;j{3*h?y^T1+k z*we?tN7OtePP;XHf8f>mG#C2o@n9na;RKJY^b;tc+Ad;NPI+k(I$(W}tfk6{4HcEY z>DptDiyJ1yb`peR!QH}?SgC?fZ=D%g0C83TNkUEvj2RWEym8stgW+@foMa7QS%ey< zx}0O5zbA9?H z5FIRrxPzJ5Ks91i{kO+*+6bqe953cC$#Uw3no+LuH9J&5B zEsBUf+;ICoXsqe|fU0uiaNu#e_n%L%2Dwm&)oJ~)Gysv0AMU(HPJ4Utx{z#D7K}Ff z&+;ibH(P#{vh6&z*F0yl=0Zr?b15oU@ZE;LdYr_adVuq0#f(;vfg_7oae^R=~+7KG!AZwNCMc$B{jSvn@!7TxX{prmw}c9(L` zE*%l35N3M%f}i+}@$?kB^`_U=70FD|OJPaQzrRDd6}e8?-e1+>7*C!3!lx3c(`0lG zx&%ymNCiZ3XrB=*sRh}Y@AgQWec1ghz}WXF<>!~8qj;A25wg{%LY{+V)fe@3?|%-Y zMZ83T8=dJ~WWnASEaA)U-xS1g*o{@DK0Cc_Lje{9(8cBO3Q4ySDNvIf-*$ePc{*9k zr6|%cD@11dty~$7HOq2tD=91)+J!TSzz@DOmaNy;o73ssW=|K(?_ql(j|psb(3HzN zZD3u36vV`*<0$kq%(+_UP-mRY+>LlOUvIpX0WuvlWcd;tchgH>{(SL(I{gE$;QD9S zq7VmAeu5x&(;Lt2E?y(7nV5~VDsiI$|LaR^VEFYF9C|at&$u1TnjdlY!v&za;JyN= zZcm!T};_&Z_*p7YrSf$*AO&Vr#rJ_<8sh&u_jdO;sZG7pq3%{r+rJPgxwFz zO_tUfksbTPeW9tPsRE(~A*PJ#Ro4UO0tWH;k+&M|9P_+Dp;RN!FPF|$kiR>F!-YK+ zV)CNo42QEemz=}pEQuIuU3@z62!L1!i)(l~t%ojcox zR_XhChiy7Ytufm~Q0&VuseO=19-G5CJ_avugyHNLOSxaVvbstKgP|LFkXLAF0l1qL z@3{jCV!0HPk$~cP1n_s`#-r9pdayeVW0aKN%F>GSi~IUzEx?fgE(3yij1FEJ_+#ct zeFS4`!bd~9w7(x7+9q1a3LhbK2bnGKo3*66HH@6x+d)BoEiLD)9JIY3B8MhKvRiw| zvn70?05{0R#aKKHH{FaVIP$nL@Vw$)x3a@{jVe%V=wN)30WJpD`*;AuJkd`|zS*)A zmu;oQi(Hjg!7|Xp`cC;WWP<1+L;mI;qPKQ-Kj&4gx9<-+CZFEA-aWW+IwOzUH*vD( zs&aBJ-|{=|oo2?C(d*=YF%rz@vpkddk-z&D^{VzGFmlK?(6N?#FCo#H)o-=8R|v8% zyBnk(q;rYmOuE}JtgiL|3{t8lm{Nt{PE@I|G;6qV z^=s$8*F)Nw3tMevP&)oMa+cryK@*8C(DOG!<8fA@bF=t!G&Xqd1riqz+t_%cYHQ}s zt2mXD5`MEad4&Q(I_4*N6O!i7@C`pReS8f?!P-GNdz&?;d zpfo;y55Db_F(}*u8$U2Sa4&uY(PQ7ei6_dt$mdx4Af60?>W9*E=3=LjibXQQ$q*UT zRpRJgJwb0;s;ZygJ3H^6_2_54@W~lBm;5081o1&qj|(DN5o)ouH*sCl`Ut^pevlv# zc5(dBZj)-Y-ep?*@YSZJMXXbCg|aK^^+3xBPFR2RGTr&)QQD>NX9u&M+Zi#pz0a#% z3}qt}wjESQemkgo(<#q+bboW{eiJ$5U+G=#C(x7eJQ>mt-(sFY)aa}&)+Ou~8&2Ow zIJc|#U0yC9ILOtG{&p~%iBq^o9}idgCOS`I0AP>5z>RpIRBj((kwy%!@R#6Ypn(Fh z7^|NjdK}%>cS9rP%9dUbMUW|Vbpa%k>f;R#Z4ithl*ZGy($OV~h<$H_fj4>c z�XS^De^q@wox32%Nsh88te=V0ZPk^Ym3ICRp##(msBk5%0(iiF|cSboXphyBoG9 z$bDU&B_GK{`kh^uU-+3q9VviJ|9pag6Qr_00Xwh@`?;fMP;f8-6uLbUKSx8%a(kzN zM6;};cD7M?ugDk8q$5zLXmykMZw<^4bfID5en@+DK|y%7P__wbz=Bj;apY}#!9T7v z6>z5Wtiz%hBZf5^PTRIFqus3ucdxABaeynPXo$0-79~|ao-g@dgt!Ptl@!fG6 zOCzl#o5n)bT=3%-CIo-Gui4ggS!G!R*p0d|f(^v<2RL8EBY z_cZc*XBj4GXCqiz5Ky+H)J2)%u%9Y~62O$=qCQ?tP6qb(NXANjCv~>-PxV$p%v-Sq zL*hsoEZxdb)5`Z8PjY=*744%g2f`fDjWXrx+Ue0w;ppJe3w6QzOu#D4b*teM3dp=K z1x7hEI0|_J>pr;KT`0{DF@37NP8cf8>9v6zS|f0bpz= zlZ|yxx3)Mmep1u3)C zp^ijIb|H~TE&{S1IU5lqgaEEs{&>00W>0fB@(X@agi-9ld;B54PY%r1Nu+DGOiax7 zYcqn0W>^Eq-jxfS(;{R}R+v2(^heL#p7=7+f50H#8oeEtN@pz}ww8qb;zj<2^yk~( zeaP|C;=*r<_ZCn7OHLg)LAspTu-jX6vK5_Ol4tEk2~Y@69fMU|XXVJ!o-6XbSLOo& zMJ7dYX)CNmYEYAL*qfGG+agH7PArL3Lqot&j@K!55d#IUE@dD$^#vzg58IT%ifnVl zWB2UD7(kPd$GODal& zP=mFQ(FiUDyJUOp*Hqh!;A2=(XKl1ps=yZPnJxj8;By&Kzl*(#Y&xRKd?GN-ND1+L z9e)TdCnpC7u0G8mJN4fqfRXukisnsr=4#tFQq&|B$GR_WiGbk9$;0(Il^JetG%Cr6 zZ2&AA2CwF1e|s6TRKn+4@)P6N&m7wJ&LgpobPoa1f7%NrghMe-ZPz{8v3oU9D)SP! zXZI=cFF?YY`S%hKE#YqRC|!ipW`?C&PhTq(jIa3$pI`Ug_ryjtq%FN^mkTTR;rav9 z9bXa%3w;hcxA0~be}#G>9#9cCPEXI)&ILsPy8MZcHhNdPu){@#wFI!J^}{4%v_r9j zAckxEi4$-y5*MnnLf_tthoakOUCc?e?H%z_+!6=bs}^(FHA_>IB6UXr??N5c*3Cu(PNqquKN;$C*jUOAtVq7Zj> zwwP_*09))T+5%g0P=I*lHy9oKXD)kte>%`&Q!=j(-4zGY=s@F`>#CTOpDX6yv8!dTykr-pQ`)SEuF@|7=tb!Rw0 zez-w8jcn=(s)CTT3gjs}GBV6Yc*i?yIA!+TuS&3>0al!z(CANOviX zAV_yeBc*hs(nvQ*NO!|U8l}6tq@=sK#O!;~7r%M4*8DZ|7V9lUIULT}`@6sK3D_s6 zmLN4bwO8?X!y{FXH0(&W2jxC2q(5$4XqvjwW=4&LucPg!E135gLzyg!*cxo){$3R} zS;G`V^&UcWG&63fcrKGRa#uYl%)p4#UY(QAyk=mJtiLbfPi8Vq}zRo@07q$M#01Iej{zPz)?IFLcZj zCX-0N1(b=B+v4N3wi~&gofyQ|fXzlLe?+BW{k5N-cppl7*j_W-c`_-^YqM5fp55vP zY6xx)CR6mlIrsLH*>(IfEF1!NkmXYo3i25?Z9cH6T>@%HM-Jw4gwgPuhHweA+=dAK z@=D+dv>T~bGcmX}tG6g_%2nM4rJHXXCnlyND$BL2#4CB_AI6xOI?>+VC15MX#x%gp zSlIAfs*~K~1_?;FhfWwBONTPc4SPs`5>^Q)YE)k9;k~VH8`Fat*Ma9|ZCyHhGM(9A zPokskTD%#?B`58hGT$y;kh^EEg90`0Cp1f$7$+uLErLQPC_lU0rGDR-DkaU@bdSgh z*=i*)ZM4t?xw34_3n>fv9wUQ$MF8l z!yB$`U15Dx)iSRo{lkiuzoNVMJ}t%x4GqO49N+G|{jpah>-S0a(nlEkAw1_gqS}Z^ zEyOnTrddE;i%|MdcaDaVfkC4jLUtJiBG6q&2`*jPk zEgBqr7lls8mANlh#>qx;E`QKF>yj zrzv{S$oExs<{ zOTc1>iJdEog2MB1^GYuM)a5SVI|mf#Xao)KfhMM3ki;ze;$$x|iKsMxWT`?KWVRk; zEqQy)(}nVn zwtQ2ICy~L-hm0uxyhmIV5ejA$%@kZ5?4>Pp3r2)vrNc+DUG?ETp+;d>-BN~XIMxnw z;dyqP7~vVcZu*cJ2cM%?b?;I1J4(XZLd?-(VWrzBkknb6eK_4jCt1f7JqV03zP8IQ zTANZ!1(x7+i)=Mcw~s-L!_GOGuYt8;^)m`RSd`4=eh@GYZ>a5#=hhADym9^A?t=Iv zfFlC8Lh0A4s(t|NJ~wlD!9I5ep*O8_9X^9P9dRX|PzW0?I5?0ncHRjoBS#(ME(sx+ zoe)Dl$*oT2S?7V`R0VGR2qlqtOUIYrl^H^uMJUcF-s;o!+_6IrJ#bVmgqUsslZ0SO z+l++)A>~`Gw)e=ihm^t?*7z%tj|889f-OSmTgOQyOclVDBzv8-caWt@(B$oG#p3;+ zciejp5;o%FG-RRW=^Ad_XX-{1=;b-yj1Iec&Vbh#&1scR9c&3c(_Lr{flH@rJL*M7 z;a`LvDaMf;N@0%SroS3+VN(_p^Qnw1X2VXg8ARF8P`Ml@8nTK9K^wQLttjW`zV1of zO9}m&AnuASJxLCe>s-)vpC?$ae$ucY%URZ#Ujf|)X+})GmSHgefpH-YrMVYfs4fsc9cH(M=FmrfdmT>$fcrZ*Jr6#0W_@F-hEsYmV(XMM0@Pnr3#;|@J2XaNH2dG z#1riIQXWtSE_!{WwF1q_?p*f-%AZUPQYe#K4d0ggv(T9#xZ=MSy|Erz{px%H#qSsj ze!FcpUi#gifSEz{*1h2SMPGW$OqAffqUDM5f@edEHLR$ekQ+RlW2P+ z$a%yPRCHNlce9$7+fqyW{bUu`Wv#3;Z7xIzRwu6kCI?g#KQ(flq?>Cidb zbv@Aj6_b<{N)ZWoH|?-AyONP80z83sfZl&{bu}^Yv<%rO19rPIJwpIkF0WENs->J% zTr5lX$-YYO@VSCQ1oicl*|&A;VN2>8BY`G$TEQa#i1o9$LmM=~Jk_9IRhGJkOO0V2effv$4?*kF$f`*+>n+(6x@R^1H?P%9C8Y*|);h zC+ghl^2`9s1YG4szPBWPVPYHLJ#FT0VLDHVe(T?q`b|qXE)L4vi2IwDVA5)WPQtao z;+aM2;&9+9X)OlpEuk;xr!-^>)^w>jcOC2Q=&)M8Vj0Nu8@MnAY}A<&2)xa4fV7O& z5Jjk{pj2%OT$Z1s-o@O#fT1`nKJZvrP?nY%Z&>atY+WeebHQYCVuM@bMh0WXeZ#*WGp&Vx4ym1(oJtPjp#P4IJ=U5=fG2bpU9HhQXs?aff(S8-b`{jb5s1EWWdO!9n4PCuTz@eF$1Z zBD5~YDtTkeUaIswry9^@(vTM>t$Lc=fSa|#k)ZjfLFAQ@0y5xNlyPB9PEHPVa!|dG z#2O-&)FJ5Mt2V+OKoTR#Sk8peXop2yl#V3aK8{7T+&nL`#_1^e+qY4u)~$QVKv9Q; ziiW0S!vZkyxzdRzE^gT__r5r{^0g98dFcmx{Nxd6K2!k}jg6@0-YhxF0|%$|TdyoP zNw{t0PuF|cxwr`1+S-83s?Vv6e8ZF#cKq%%5WPwPW@@DIVZk9GNf$9z`iGHUmaG?+ zv7FCgBcma8$XR)E3eS25!v}9A%9$OB6O&@mk-z&k$KX1B#;mxhwW9r+E9GnHwM*E! z8#)Y>#^HjGvW&5H8gxSN<9$4y+z~TR@UM-BJ!-~w4^4T@x)~N1MEjWXW#M_f?F8o{ zS$j*3;q+&5eo0Fe=d)?jqvXuO)_JH>B8;LX)P!zC*7fmmp8`K+A9W&mi-d&4 zp6Y~2DLb`t5`wj~LTiA7Tn$K271F9RSU5N-)%&&UEyCAsMgZ;1ZmO*FT=+95{;W zxDPgg@zLfK^l-L=FBxj}Qe0f-k{P^<5`rQ?nmBaVQ(54c*f9O-6&FT%ByK`xY1Wi5 z2w&eBfPpYJo+Wv$AFwrrC6&OAS~ zTpoy?9*NdY?HqiQ)=?UBb6cCjF!n(spGbx^`VN$A9{`ou4GYqVOBnA=PSKly>kFPv z?+=!g=F5-@W`7~E>jI$#<(Lxe3Dk03r|-7wRMQ_Pf2bdDDG7OtlIN)qapB;D> z*3Q2!L0^+<-g&$;Phwzw$K`bY`_~ul+v_DgmAt5uRXhebxgkI9JuDRveN*uZTB@j)W(^q*phN+p&qj~>zPHlDfDJ)=YP8-NCvy zi=Nu4u+q#->G?tV`J(9MH67@KC1h*1$Fck+|S2pjLW13!;Au&X9M0> z@9$%14~6(6Yv@#E`UK|ZRVAxUm@iLeGh@f9k~_bxx>YRQoE}Q8`i||zR!LuC7GpN| zlF&rr-+WQ;)B3}s4wT?kpQx8yuZ{8Xw494}6@BFz`R;<*)m)#Yu)KV@YBo8$c-42p zLoNJ-^#WIQCD`Jxj=A-NgQ+cF+YPcGL~u&c>bM4&b&pXO*EwzkfVt^9XJt#I-^-jj z$kVLS7Ek@*7{fA1AJLqAuu(u}Ar)=AhWwI~Ip2~&Mj3mxOQMJ!SwK#urf2RXt=^Iz zIWw=EaHD+BUiOkpF?(Rm)Zyfq#SE5-)NQ44jg6yUF8$V{bv;ThLT0dob@Is-ZhtPy zuCIf`2?=48@GV)7dt9%vxzn+124m&CzE)D*)l>aMD zSQI&c2>Ij$Lz^iM`}XS;E{60?*7FK_YZAh zSe7ekC`H@}9ngpF=il9B!sGls{O@@v5?MR#sS}-^n=Kp(y95eO%&;GcH)o-T?Ylyy zUVh;-t)x4U7wpdBN>fM!L)zya@1IQ`Oh~G`*DbYa$8JBCS(_T`dtIs@>Es% zZZDwpp@8H}w&!!kO9AgxMV~qd+KV4y*-rfK!rs|6j0K0|MC{!aSj-pENyNl++nBn& zV{17$6CA^ec7mTr3(eT26`3_zygx0A#V4qm7!}~#%bH5j3xQnMy|=_cIwH1ypt~OD zQ%!uVj5h%SK#iy+gSmo{X7*PAr^50*N&Lfc){CFWCNA}4Za09SsX-4E7wK}9X6;c+ zunp8|Ka_y9B_w#;PS@<~h7TG%GH z0;SLoolW2ea!^aA-}_b0#-aaRV|@p2Y|h;Ry3yKe1qH4((=EY1Xt%vZSZ{lC(176> zk={b(-Hp7CR``St1v3480$Mo>trL>;39GMt=xE8rU`EJL9P9G5p?t=zoJ*h)Qf!+y zzgOd^5)Uc1Ub7t!nWIUKKIxzbGNY(QC#e&qNahtym4n!x(mwpjuL4VgYzsx3y#ZUO z*$R)}Td>JHx7&YC4n05)JqUD)Mx2~!va;8sa26@Izi*KZo=iWUkL2i*1*MtCF)CmV zxvpHRI2<>DyA%@hIV*GWJp1Ve)pl-&nX!*zl{(WcJBzOSHe=>;$t5`1Bkuq7BLx(B+Homl`v2A?4yKAeT?IiGv9zl8HJQh?sKez`dc z2}3>MlnkCMl{_(|I9GWVUpO@6Jr@&52p_wBu-hzg22L`=EJ|8#E~m^@c1p(i@6Hc~ zSu~LPr!&d7TQWDDH8o`t2wS>4V!SCMd1rSR(}L~Qt%>Jin^oco`*t$O5>*!*RM041LtP`E|dom9?Q7cs2y^2R2_mzQui+ElS``ba3f||ov)iaVE=?8dt z-1lAka18m6Cze=kyo`9|EtgBJ=9g{P=H&qMPeDQdJ+03?Ew2JVq$`Ar3l;QLaXn^9H|ZE6@P?AcmjI8GKJYt{l|>5@Q60;PQsXC zgRYe$V4z^*tZp5mL@w0{9!|q$OJ*5l2_pO>XC!!1;oVyo=O{}ic~xT6iQy&!vz~gD z9YanIronkRsX)a5H3&=UEdP@>eEDvb;`=@&4Jwh>V=^3 zJp2dmHN+vtClC*gUMSu6NS$ib4*@VN%VA!au=rTb1IE+25_`Yu)1p*U1ijFZW7%xP znjSz6F+xjqUxpk~CKRSGh2AlM9jP5jygB5*9E+{YF;PnCj=n*FmQ@2eU|VqLihQo|y`J}1L@;68(Krn$7HMR4KQL%G75dW{owVX#emkpu|Uot;5d^K6Db z&=;Vu*fP8tD)n!(&Ns!6Ju`?K0(%>5aGw=lU!?rn@yhVX>k^U5@enQ7JH#BA~fZR}9GKjGkImWmI z^d(r`DZ!w*D|0p%uBht~{FD4O$-_!=!S6O}xlNrjP6eAI68C)vZ$Msrrf<;;6>(Ub z%^S6}M`a_MHr3c=&oRsDn|;|FC1XLq}hN4(N_VormpMrZj?j~%&g|DFIO`>D_0Hf2_QjC({ z6kGUngah@X!wH)g{VD@G0Nfsk5A=N>3#s9U&n2V;KRmXH;NW(Ayf3@9X;wwfMBD$1 zBc|d_y76x8Td{gs6_prOqC247HHJXq{M7^wTcfeztv;!|W@0fYb2&HPB28KjST*2q zBiXdK>I|bl-%OW@S6<6N=NQilBfiky_AIx;kSKqf)tc`(1 z@FT%^)?bMMh^E{3xt>e-Ad#s1puQj^>3Q)(qBmAFqI^>nQHKw58<<&vnhPR9gvqPTokuTpS6 z>!|xVK{unbhGlX$2aFljNu`MijPv?)yTSPgXM}pTnn)-#F;c@f?B42|)+{;ePmwKz{s=7Oj2e~$t|5>UoxaNz>3TSH@Swr=#f z`X{h;opR|o+kS{Rkmgk&)03S_niJ%Zw-n2J+!S_h-%HT)GH7;c-Hzn&G0Uq$E_PL$ zVqu$Bt}gK*QQx!^uJQ}3YNIjEE~ zfP+`BS_pmoX?1T(-lkUIUhxLjgxA3ldR_dVDcG^!bGlFF#|QDOsL>o)qq;}Bc9z9p zpkt}Gt$~5F{4Rpz`Dgpg1jlxQv}!==TI^=aERznK@m_3vSHaQ$A=s|z zUyHGoYbRB=)qg|q@jsol(zDXP{akA_kcy^>zk7XGVz*|sHy8zt(-r^=R*O>A^FM>E zWm)#BvH4y2qc^njGSUL$HP8g0p{2Und0_1-uc)X4YQ?u^6z~&hmjCZe!iP+A27oFK z*SzU!oN~%Sy5VyockBC+xN~`0Gug}s;T&AeT?F1$b#+8NXWRFt0YkoadY4skQ^nUz zw0ta%Y_2glT;8xb-WQcy-G|Z8|9ia4W@DIpLgSaH8ebKqo*#ZYG=(RJT^yD(SmC_# zUM_Z94dY)%bcVE)vhK^N4vO38xq*sV<8#n$Cm2Hx0*;C}-Vxh27fFYy(dhoX?A-+J z9OOsMX*>nx@1M#mD0~NF8}h+f6q{%XcLWLu@dpk800g{y#jaD;7(g13i&!-K0M zE(oWFU579K&gREgYun?78X7_(B-C8l*gC=n39L2QTSG?p+RUL+(ULa1sVj~IZ9{x#b2 zEE@^OkXL{P#L+A%vV_jryP ztiH`xalaeNP@0V^?|$)RX+qm#cA6__AWMPLpVx}^g7sJ{+NH}Bj?;cgO8cFx{On`T z#TGLU*S3thRWs`{)=0{WZVa^dmg~{qkWpKnHKMv`ykk}I4v zBAo(2=ZEhi9AB<#w_a4r^2Jl}C&l4m(?L2w?>;XGHE*JLP|?1c&&ORCo+U7IKbLjD zW|C!AwsVI^=f+onu8xS)9+)~3m*IbAU`ZAsCMTxi2Mg2(;e$2bHmW8t!2})5#4@@A z&O)8b`6M1OQ3QlTDcB{TywgBG(CzZqmJKO6GEW5@?Dd> z91P@pYrDN#b>h9ga~T9iq@8~vCi%AYeS4#SqJ#UG;5Au855VvLwTW&sULM%6AIJkZ z4bVFh%R@(Vm>zromYjt`URX~^R7kgFhs37#A|-?@%Mrg;D2`}u;hm4qN7hFM2-e&~ z_Kr(3kZ#1V{!TUE*?`CEpy^l6f%apTR#ro?h(fR-@$WtY=H<6qfIYRNIT#eatU^(8 zb7eJ8bmAseJH^-3I_Kh>+2)J{cpRYo@cy{*&6frtVnPiyJK@Xl3;ZcF7IRJ*t|S4& zO~8kJkQi`rPokOb`>EQ<{7>H7GKa4T(5Ia+IqiN?-FC;-mmwLW$0Bu1OTeaV3H0SI z1nN!Iy+=38y5plwqW2nTFWtaq|L3-Eki$p%vm9QCy)Z_?HPiA?QmR=UUc-rf@-R<< za9pHZ804G8BYhd=x?dU}@swT&zIqCMb|69l&L8f$w1m)Q)Ed8HlURyz3ZO4WkVR1G z%7$2AJH%Y@Y8}4bC{TidE(gevs^c7@zcAwgR<6v@`nxEC6oVUPt4|TPFqa6}X}86a z2$+AL83^rwe+PT=K`df>*y+nxfj1{`sk~dx%=Q$hX`Gl;^|e82>|A*vr%*kqa*pm> zi+S54Au9t;;sVXP+$kt7?|>s*putN-D>`jT4o;|oOB4M2^5851Xw`HiEP$N&?O${3 zokuDLt-H^nRbx@+w;Nq;Yr=GQr!jW18pGTfS&lj9T zeZLHo(&DMym&u(hk>%v?HgdYP$+<;JTm1$SIB9o7E9iujb3K)LAN_4MmT4 z8>p7NRmO=B@iOR7A&$aBk{|#ut6ACEKY(=T;QzijgHMIcX{T%PEiSgx0jY&1)x2dl zzGLTS^6fqte+|~94i7xX2pVzm2VI#Iz@+hC<+^L3ae-_5YTT3G`~7V6u0ppj76icwO^jtBu{ztu$tvFy2cJX!y&j>hK{=i!2V zTM@&ppMf~$L>$I$+)I-PW(?GF`o^99dGwa$h+fo&ey=z7&F|vASU-F8;}K!1@T#d; z(%uz@0blhha(Z9_B-QW8DR569r$(mc_;~tL4gSFxm3rUGT~mH22_P+6+4Di)=Td7v zE92248DWt!T)g_8Xflx2ONoi00Brt$?Bn3^5EdoCcb+kwcUmSQFgSJp2st+r(53Gb9Xmb12~+&XvN8|fNCc4`(7NQ8)<`!m8Q_h- zqE|XLeZU?8+>eh6V4NCaS%;BdWP#Ewixq)z@#Z)F(_&eDSQe%fy)?X53Ej-j0o(g1 z29YpG{GCAmDOIEt6dnLY`GR41jtMBQsR%_KM8j%ONwpTI!H|JVkr26`hQ$3QkTWRSM)O`VWetE&T_@)q;|OaTztCj{56Ku4#4b@%GW zVqO*wb!o$t6cuz%@)xriDhJPkOO^cLdZHk@e3t^LHy8PH$-w(DZ9P5g)uaEFS8S4Z z2ZB0ye$M0b(6|`bO`HGyqd~n&e5W_W%fR|8`v1JQtMmlE1^|Lu;m@f9UjlzG%-@d+ z={CvL-TnRz03?Cq_XEmq{ha&z8sdMh@fwb7l0fvSW()pt=GnjWX_^VMk14I>xq1bS z{H8YJ7graJ#sR@c2{rN; z&|fXcy0^a<L>UN3&DOMREX=&}@=_!L~)*J`?iBv$_ zH)(8nIyOLLA`x+dk~91{L9g1Ehk$*36U<}Z#>d5(mi~4X^g;ciarcqqJjZo2H^eC- zaJ}OY18g>IdNN3p#^J_TKx8(aotUBKgGx`1L=@{k=^92J_85`W&`jmW@Xq@x$KpQQe56~vPY{_u~K?CPc6AO4fWQt6zORdfJ5kfp z%5N||ek^Rvd3mU`biIhRJ3==|hwM?RwW2b|o-)53<`BPs>3!bjhIw`rOs| zTz-Cj?*L{j1t0*jFPDe&h^Fo5(F)$a6K(m3d|RF=R9I(v&_ zr3G$uBa;m-CTz=K& z4=@M<1p2xp!=40uE^8{_ah;hHI5($rFvj$d*)L5ZD%-6%f<{T_`=^_$pVjT?BG#@ry-m)-6Mk-Wn%65Jko!AJ8Z+=g%(pr5q0= zUU+~BIV;sv;TvQi^e?P0w&Fs+Uf@}(Fnf8ib!k1c1BUt8NU)rO+H8NkF;}>7c{@$P zbBl}9k-F@)?K+@*;dZv;eOmWmUh!6*$bpI_ou=!FzWz6#5IT|g{k&cM{0wx5i%+P} zULUnY4W_36fQCz5>sOUwCI%dG(R-S_HYji{Uo*vV-+l7kluxBmg5KyW*@C)<>tn0j zqW+RvKLgPjO*dfSKnGCSj${IACbuDfD^o>9woFp@YFUZX|jRM^kn~KcD-X z^V;}muvtv=I0JNFwp#?5gWl_X;NQ8DF6i1Cg471W%FbexmQGhJ>#FyY6}1{;yb4{X zS3nIy%k7*MNX=$x((`}yC+6%oA^e0kevy&1u|)`1!p#u`)6Tjo@V=_Npmh(%F9LfH z$*`57b)M0od@iTT=2aVP6Sn&LX^$Bh`*s1gjwoF`wgZrF=tQixf4)f52OC91qNvn# ztbg}#!h&db3I@~wM?l5rd|BRz>s`hIEP&Euw2TwcKFfac@JCezk) zF$swh(4Sd3t%%Ze%+mkiaR(g+wpRc2;E0wcZAF#(N4>N}3H5w*t`vK#<59*_f;tcump* z+WG}*-`*u{a)I}tEB^9jG4ONm254oS7%rRa9+q$Yj6k5W1Gr5)pH^%xfBO|{LJO|W zsxE4?Fb@1D!JC?@vMcAgIG(ROG#;`6u6rsQVTZ;R*=G|d6ixmSW@fiXvtT!m3%5|J4lCPN`>!;=zhU&JUO$9zH_l&8}`sh`#Mz(6&;;| z39MGZv=N*;iHCVZX7ojJ`UcT1Zqy|=C(12)E=rMl4+}`fFaCO_ZX@6eMDS(JY6Khc zfbw5&ho8|N3rkBh^N?rHy5H29cHr4FkjEE$#EXya5g5iL!ImU^(ibYjxb*|a^V6)!0>?*5A9haw^DLU*t(FOQvwBae z8mwrByNuX@B;4t{EUR7ofT0Q(^Iz={lSOX*`uf69FF2brlv~#IVK8wcHsMp3*n9UD zJ7j(+2RvWnA1mh)l|>TM{K=@>oTvcJBnh@C=*RZ(_RisJ z6r;Ft;f}%gn%~Ih$sNW>GNy$~F?x!Pz=fKKefs1jxT~*%l1AE(IN=}}v6*M*pRs0# zL#2dj_a4&$QSsEuXV}@(b#87`EJnxyZWnur)M)-^Mi0&OdJ=f~*3{z1H2V$PSu-!( z@4yDUUQEoemY(PJ9`d7^xrrhIgxdyurCMN^UB>nvUdlZmyO~0*Vpb?N78ww#|F1G&NG0v2Hh2506u?1nv34gW7?iNQYe*l z^7SJdM-601EAca|@H&-n?FxE%-Omo^Q}l~%?<_>6uYO%#s}Ssb6uPC*+S%C%qI2fy z2omzGdK8M=g=d+%jYo<1K2gx;cisrw;(n@3W$U|N;Mmm~Nmfo_Snk}@?oYd-wpg;r z-J*2UA^=(=_Di)_m_ki)=)WDQE|d-nXxlDX}9d zwfqjnG(~oLiN1p%mSK=57HB|D_4KI>CuAdpEvajnq5WHgEd=rvz-UDj=c1ooIgh#Ckva*~mZ|j={3pN>t_|#P zvX`d;p3#Qgbfcy_bWNcapJM+d&!XeQStaa#7|0v4C*4$CIZu|NlHWQ)IFBegCA-xu zr~IDW5g&wg$mG8`Aa2#s^V>-_G+%JqtPpkhvDoo?w^|Am11{KY@jp(6Wh6MhER(zAu2o_X5L@w-xo zk}!#3+~Bwpyjavi+5p6*2Ms%dBtU+Z7z?4idFP;-beEr|B#xpdFZF5gm2 zN?cx50XNFTU)R+=am>-kbddKCMW`j`6Z)J%h}1E!o*~?LrD|W-Me??)q-nNK)xRd4mVX&=j|5~ z#>QEjs(k1Bat9~kny5iE(O6TI>?4kpl5OC|DC@?mj z{8l6l3$C;VWr2!$^^VAp;-aJV=H%Eec!Sxl!qOxl{+E}Uj7XJXtgMj%bF_k^oy}3r z-8!eWymD2*o-#HD?klJKktRZf=w@y-rRQs2@Ztt!&86p4LoX2Mz{}H>uNy%?ZeDO9 zs80(2-{mT%#Sg;&=%f5@NB!j=D43X;v5a_5ms5}&G!ft0IN5tRLI&^J%yGi6??Yw< ztevu7v@-Yb%on(~?O_R0tb+5uUp@PfawCuB?tod2|eq`ZF5lX7!&zi&-J&4}QbIk1=?vat;> zuIbf>MMW(E<)WPy25xY(c?Z>qzJS=zs~rSxboCQ_e6(ZrtaAAgs0rplvYoHAyk828 zPGVzM8JJf=W!G4DaX{bL)RZLr8P^_M3)n1t1Th{jPqj>-%64OW+G&dujTe}H?hSod z_h;^|?@yB;Nzf`cC-K4{%fDyOc?eH!&(54D_K32lVrbTJWLKcAoTMSd68|Xnyg8Hm|yWoP8Nvski1w{s3gz(-<`&T~a<3h7S z2K#O8IGbS=E_re6Jr?+=EQxQXXMp{s-(MCN*oWKq52CFcURWy@#!S|2sP5# zS5O{<=7;-B;fF^8UwogGbczHJrdfjrkn#341v0M_yk=y57&3o3Pj(5^wsDF8LG)mf zzV0k}m}xoOst>y6^oj)@=WKW>46QPx;s<|9-1_#ztbED~xHs9%TprJ3u;#qJm3y^G z)PFMxz(Vx(1e;`r>30P=I1hLps!t@NzmkIZG@&!o1R`Kle~F>A_mNaQ*AS>W=ibqd z<>%&>ZmdmKl%#3Zxk5Mm{O(Es_V40+5WnIC2$ri>2@byqCQLv(hz_*)N_b%FC2wJ1 z1t8q+W=nvPUEXv#Xl<48YS&-uP%(^GSUP#~#q z6Tow?5FS*@G?@`GONSQjz61pPJ%UehG_}0pU3jZU%yxQE1wli;J527>Ec@2)zINJ& zZkwaUEK+b9VUY4HZ}Y{Rq+tOYEl|-CA+BpOlHU~iSR$^QI&@5yiCi3@vhDK5!ErTs zuZ;hEMH(HZuf~^BD<*af>fy~oX@PTAJ&|$1{`3IROK$Og0;u;lIAg>gzgdbsT=gN1 z33V0-sbnT-N*3KHu%D10F8hVDaXZ z-eq#?x;Xf2Arz3oe>oh1Pk8VENAY}lrmpVp#|v<(W?7z&lOq$Yr6v!AM@5ZrhzJRJ zG(Z=EY@pKq4t7t$2=2`elUPomfj?@fo1*l>UjM*=iAxu}{1_GuB$oM{E@KHBw&@Oi z_WTy!knv0{uYdgoUpw#pQrPiG-GxF~JiNF@^PtEm^%$Oc-$LSLbcK;F6P&1XMJ!R2 zJDSSyGvPW)n=lMSC+_Y8V_Y&rv&Qia@a)SD zPQ1sp9(8)pIhNJLtFcii!fAwbY!mbfs{=Y;f8lXpns!{Hivy7!UhbLoK4ea%z}p4U zF0NLoTg4JtiC;@a&o-uHE#1sy0tnxc>R%P3Nz4t5i zgSJZu(AH-t)!N1oK?4s zJ9o@`n;Dm*v0FAw5~iTJ_x9aDffheJe#1N0gJp%H-MXMcSv{dlbjAk7T_~ZwaqCx! zTPjR~5}NZlCXXuh3#m8Y9cb#hqEfu8ZEp^`2e%O|FG`OaL=f3KQ$|=|F^UyJn5*>| zs8D}T2i_%Z0NIe{? pd.DataFrame: + col_map = {"benchmark_number": "id", "cycles_per_tuple": "cycles"} + df = df.rename(columns=col_map, errors="ignore") + required = {"id", "name", "iterations", "cycles"} + missing = required.difference(df.columns) + if missing: + raise ValueError(f"Missing required columns: {', '.join(sorted(missing))}") + return df + + +def load_and_prepare(path: str | Path = "patch.csv") -> pd.DataFrame: + """Load CSV and return DataFrame with exceptions, cycles, version, and type.""" + df = pd.read_csv(path) + if "benchmark_number" not in df.columns: + df = pd.read_csv( + path, + header=None, + names=["benchmark_number", "name", "iterations", "cycles_per_tuple"], + ) + df = _canonicalise_columns(df) + + # Ensure id is integer + df["id"] = pd.to_numeric(df["id"], errors="coerce").astype(pd.Int64Dtype()) + df = df.dropna(subset=["id"]) + df["id"] = df["id"].astype(int) + + # Extract exception count (trailing digits) + df["exceptions"] = ( + df["name"].str.extract(r"(\d+)$")[0] + .astype(pd.Int64Dtype()) + ) + df = df.dropna(subset=["exceptions"]) + df["exceptions"] = df["exceptions"].astype(int) + + # Extract version prefix (new/old), drop 'try' + df["version"] = df["name"].str.extract(r"^(new|old)")[0] + df = df.dropna(subset=["version"]) + + # Classify type by name suffix: 'float' or 'dbl' + df["type"] = df["name"].apply( + lambda s: "float" if "float" in s else ("dbl" if "dbl" in s else None) + ) + df = df.dropna(subset=["type"]) + + # Convert cycles to numeric + df["cycles"] = pd.to_numeric(df["cycles"], errors="coerce") + + return df + + +def plot_benchmarks(df: pd.DataFrame, save_path: Path = Path("benchmark_new_vs_old.png")) -> None: + """Plot float and dbl benchmarks in a single figure with two subplots and save.""" + color_map = {"new": "green", "old": "red"} + types = ["float", "dbl"] + + fig, axes = plt.subplots(1, 2, figsize=(16, 5), sharey=True) + for ax, typ in zip(axes, types): + sub = df[df["type"] == typ] + if sub.empty: + ax.set_visible(False) + continue + pivot = ( + sub.pivot_table( + index="exceptions", columns="version", values="cycles", aggfunc="mean" + ) + .sort_index() + ) + for version in ["new", "old"]: + if version in pivot.columns: + series = pivot[version] + ax.plot( + series.index, + series.values, + marker="o", + label=version, + color=color_map.get(version), + ) + ax.set_xticks(pivot.index) + ax.set_xlabel("Number of Exceptions") + ax.set_title(f"{typ.capitalize()} benchmarks: new vs old") + ax.grid(True, linestyle="--", alpha=0.5) + ax.legend(title="Version") + + axes[0].set_ylabel("Cycles per Run") + fig.suptitle("Benchmark: new vs old (float & dbl)") + plt.tight_layout(rect=[0, 0, 1, 0.95]) + # Save figure + fig.savefig(save_path) + print(f"Plot saved to {save_path}") + plt.show() + + +def main(argv: list[str] | None = None) -> None: + import argparse + parser = argparse.ArgumentParser(description="Plot float and dbl benchmark results.") + parser.add_argument("csv", nargs="?", default="patch.csv", help="CSV file to read") + parser.add_argument("--out", help="Output PNG path", default="benchmark_new_vs_old.png") + ns = parser.parse_args(argv) + path = Path(ns.csv) + if not path.exists(): + parser.error(f"CSV not found: {path}") + df = load_and_prepare(path) + if df.empty: + parser.error("No valid data to plot.") + + # Compute and print average cycles and speedup factor of new over old + for typ in ["float", "dbl"]: + sub = df[df["type"] == typ] + if sub.empty: + continue + pivot = ( + sub.pivot_table( + index="exceptions", columns="version", values="cycles", aggfunc="mean" + ).sort_index() + ) + if "new" in pivot.columns and "old" in pivot.columns: + old = pivot["old"] + new = pivot["new"] + avg_new = new.mean() + avg_old = old.mean() + times = old / new + avg_times = times.mean() + print(f"{typ.capitalize()}: average cycles per run — new: {avg_new:.1f}, old: {avg_old:.1f}.") + print(f"{typ.capitalize()}: new is on average {avg_times:.2f}× faster than old.") + + plot_benchmarks(df, Path(ns.out)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt new file mode 100644 index 00000000..328e7879 --- /dev/null +++ b/cuda/CMakeLists.txt @@ -0,0 +1,44 @@ +cmake_minimum_required(VERSION 3.18 FATAL_ERROR) +project(cuda_info LANGUAGES CXX CUDA) + +# --- CUDA compiler info via CMake --- +message(STATUS "CMAKE_CUDA_COMPILER: ${CMAKE_CUDA_COMPILER}") +message(STATUS "CMAKE_CUDA_COMPILER_ID: ${CMAKE_CUDA_COMPILER_ID}") +message(STATUS "CMAKE_CUDA_COMPILER_VERSION: ${CMAKE_CUDA_COMPILER_VERSION}") +message(STATUS "CMAKE_CUDA_STANDARD: ${CMAKE_CUDA_STANDARD}") +message(STATUS "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}") + +# --- Find the CUDA toolkit (modern CMake) --- +find_package(CUDAToolkit REQUIRED) +message(STATUS "CUDAToolkit version: ${CUDAToolkit_VERSION}") +message(STATUS "CUDAToolkit include dirs: ${CUDAToolkit_INCLUDE_DIRS}") +message(STATUS "CUDAToolkit libraries: ${CUDAToolkit_LIBRARIES}") + +# --- Locate nvcc and dump version --- +find_program(NVCC_EXECUTABLE nvcc REQUIRED) +message(STATUS "nvcc executable: ${NVCC_EXECUTABLE}") +execute_process( + COMMAND "${NVCC_EXECUTABLE}" --version + OUTPUT_VARIABLE _nvcc_version + OUTPUT_STRIP_TRAILING_WHITESPACE +) +message(STATUS "nvcc --version output:\n${_nvcc_version}") + +# --- Echo common CUDA-related env vars --- +foreach (_env_var CUDA_PATH CUDA_HOME CUDA_TOOLKIT_ROOT_DIR) + if (DEFINED ENV{${_env_var}}) + message(STATUS "${_env_var}: $ENV{${_env_var}}") + endif () +endforeach () + +# --- Point at your subdirectory source file --- +add_executable(cuda_info_dummy + ${CMAKE_CURRENT_SOURCE_DIR}/src/cuda_reader.cu +) + +# --- Link against CUDA runtime --- +target_link_libraries(cuda_info_dummy PRIVATE CUDA::cudart) +set_target_properties(cuda_info_dummy PROPERTIES + CUDA_STANDARD 11 + CUDA_STANDARD_REQUIRED YES +) diff --git a/cuda/src/cuda_reader.cu b/cuda/src/cuda_reader.cu new file mode 100644 index 00000000..d60cb546 --- /dev/null +++ b/cuda/src/cuda_reader.cu @@ -0,0 +1,19 @@ +#include + +// A simple CUDA Hello World example +// This kernel prints a greeting from each thread on the GPU +__global__ void hello_from_gpu() { + int tid = threadIdx.x; + printf("Hello World from GPU! Thread %d says hi.\n", tid); +} + +int main() { + // Launch the kernel with 4 threads in one block + hello_from_gpu<<<1, 4>>>(); + + // Wait for GPU to finish before accessing on host + cudaDeviceSynchronize(); + + // And return + return 0; +} diff --git a/data/include/data/fastlanes_data.hpp b/data/include/data/fastlanes_data.hpp index 6f3eb4b8..ce76073a 100644 --- a/data/include/data/fastlanes_data.hpp +++ b/data/include/data/fastlanes_data.hpp @@ -7,6 +7,7 @@ #include "data/example.hpp" #include "data/fannie_mae.hpp" #include "data/fc_bench.hpp" +#include "data/galp.hpp" #include "data/generated.hpp" #include "data/issues.hpp" #include "data/public_bi.hpp" diff --git a/data/include/data/galp.hpp b/data/include/data/galp.hpp new file mode 100644 index 00000000..5ed6bed1 --- /dev/null +++ b/data/include/data/galp.hpp @@ -0,0 +1,64 @@ +#ifndef DATA_GALP_HPP +#define DATA_GALP_HPP + +#include +#include + +namespace fastlanes { + +using galp_dataset_t = std::array, 12>; + +class galp { +public: + // FLOAT precision + static constexpr std::string_view float_galp_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/float/galp/one_vector"}; + static constexpr std::string_view float_galp_single_column {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/float/galp/single_column"}; + + static constexpr std::string_view float_constant_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/float/constant/one_vector"}; + static constexpr std::string_view float_constant_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/float/constant/single_column"}; + + static constexpr std::string_view float_positive_inf_one_vector { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/float/positive_inf/one_vector"}; + static constexpr std::string_view float_positive_inf_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/float/positive_inf/single_column"}; + + // DOUBLE precision + static constexpr std::string_view double_galp_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/double/galp/one_vector"}; + static constexpr std::string_view double_galp_single_column {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/double/galp/single_column"}; + + static constexpr std::string_view double_constant_one_vector {FLS_CMAKE_SOURCE_DIR + "/data/generated/galp/double/constant/one_vector"}; + static constexpr std::string_view double_constant_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/double/constant/single_column"}; + + static constexpr std::string_view double_positive_inf_one_vector { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/double/positive_inf/one_vector"}; + static constexpr std::string_view double_positive_inf_single_column { + FLS_CMAKE_SOURCE_DIR "/data/generated/galp/double/positive_inf/single_column"}; + + static constexpr galp_dataset_t dataset { + {// Float entries + {"float_galp_one_vector", float_galp_one_vector}, + {"float_galp_single_column", float_galp_single_column}, + {"float_constant_one_vector", float_constant_one_vector}, + {"float_constant_single_column", float_constant_single_column}, + {"float_positive_inf_one_vector", float_positive_inf_one_vector}, + {"float_positive_inf_single_column", float_positive_inf_single_column}, + // Double entries + {"double_galp_one_vector", double_galp_one_vector}, + {"double_galp_single_column", double_galp_single_column}, + {"double_constant_one_vector", double_constant_one_vector}, + {"double_constant_single_column", double_constant_single_column}, + {"double_positive_inf_one_vector", double_positive_inf_one_vector}, + {"double_positive_inf_single_column", double_positive_inf_single_column}}}; +}; + +} // namespace fastlanes + +#endif // DATA_GALP_HPP diff --git a/flatbuffers_schemas/operator_token.fbs b/flatbuffers_schemas/operator_token.fbs index f9130374..6ce564e0 100644 --- a/flatbuffers_schemas/operator_token.fbs +++ b/flatbuffers_schemas/operator_token.fbs @@ -207,4 +207,6 @@ enum OperatorToken : ushort { EXP_RLE_U08_SLPATCH_U16 = 152, EXP_DELTA_U08 = 153, EXP_RLE_U08_U16 = 154, + EXP_GALP_FLT = 155, + EXP_GALP_DBL = 156, } diff --git a/galp/CMakeLists.txt b/galp/CMakeLists.txt new file mode 100644 index 00000000..f83fe8e7 --- /dev/null +++ b/galp/CMakeLists.txt @@ -0,0 +1,92 @@ +enable_language(CUDA) + +# ──────────────────────────────────────────────── +# 0. Global settings +# ──────────────────────────────────────────────── +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_CUDA_SEPARABLE_COMPILATION ON) +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr") #todo FIX ME + +# ──────────────────────────────────────────────── +# 1. CUDA Toolkit +# ──────────────────────────────────────────────── +find_package(CUDAToolkit REQUIRED) + +# ──────────────────────────────────────────────── +# 2. nvCOMP (required) +# ──────────────────────────────────────────────── +find_package(nvcomp CONFIG REQUIRED) + +get_target_property(NVCOMP_VERSION nvcomp::nvcomp VERSION) +get_target_property(NVCOMP_INC_DIRS nvcomp::nvcomp INTERFACE_INCLUDE_DIRECTORIES) +get_target_property(NVCOMP_LIBS nvcomp::nvcomp INTERFACE_LINK_LIBRARIES) + +message(STATUS "nvCOMP version ..........: ${NVCOMP_VERSION}") +message(STATUS "nvCOMP include dirs .....: ${NVCOMP_INC_DIRS}") +message(STATUS "nvCOMP libs .............: ${NVCOMP_LIBS}") + +# ──────────────────────────────────────────────── +# 3. Thrust / CCCL – prefer toolkit copy, fall back to CCCL ≥ 2.3 +# ──────────────────────────────────────────────── +find_path(THRUST_INCLUDE_DIR + NAMES thrust/version.h + thrust/device_uvector.h + HINTS ${CUDAToolkit_INCLUDE_DIRS}) + +if (NOT THRUST_INCLUDE_DIR) + message(FATAL_ERROR "Need Thrust ≥ 2.2 (with device_uvector.h) in CUDA Toolkit") +endif () + +add_library(Thrust::Thrust INTERFACE IMPORTED) +target_include_directories(Thrust::Thrust INTERFACE ${THRUST_INCLUDE_DIR}) + +# ──────────────────────────────────────────────── +# 4. User-tunable options & warnings +# ──────────────────────────────────────────────── +option(USE_CLANG_FOR_HOST "Use clang++-14 for all .cpp files" ON) +option(GALP_VERBOSE_BUILD "Print every compile/link command" OFF) +if (GALP_VERBOSE_BUILD) + set(CMAKE_VERBOSE_MAKEFILE ON) +endif () + +set(COMMON_WARNINGS "") +set(CLANG_ONLY_WARNINGS + -Wno-c++98-compat-local-type-template-args + -Wno-c++98-compat-pedantic + -Wno-c++98-compat + -Wno-padded + -Wno-float-equal + -Wno-global-constructors + -Wno-exit-time-destructors + -Wno-float-conversion + -Wno-sign-conversion + -Wno-implicit-int-float-conversion + -Wno-unused-parameter + -Wno-unused-variable +) + +macro(apply_warnings tgt) + target_compile_options(${tgt} PRIVATE + $<$:${COMMON_WARNINGS}> + $<$,$>:${CLANG_ONLY_WARNINGS}> + $<$:${COMMON_WARNINGS}>) +endmacro() + +add_compile_definitions(FLS_GALP_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}") + +# ──────────────────────────────────────────────── +# 5. GALP core static library +# ──────────────────────────────────────────────── +add_subdirectory(src) + +# ──────────────────────────────────────────────── +# 8. tests +# ──────────────────────────────────────────────── +add_subdirectory(test) + +## ──────────────────────────────────────────────── +## 7. Benchmarks +## ──────────────────────────────────────────────── +add_subdirectory(benchmark) + diff --git a/galp/README.md b/galp/README.md new file mode 100644 index 00000000..3644d519 --- /dev/null +++ b/galp/README.md @@ -0,0 +1,3 @@ +# G-ALP + +Heavily modified version of the original G-ALP codebase at https://github.com/cwida/FastLanesGpu-Damon2025. diff --git a/galp/benchmark/CMakeLists.txt b/galp/benchmark/CMakeLists.txt new file mode 100644 index 00000000..cfd77373 --- /dev/null +++ b/galp/benchmark/CMakeLists.txt @@ -0,0 +1,46 @@ +## ──────────────────────────────────────────────── +## Generated-bindings library +## ──────────────────────────────────────────────── +set(GALP_GENERATED_BINDINGS_SOURCES + generated-bindings/alp-double-decompress_column-bindings.cu + generated-bindings/alp-double-query_column-bindings.cu + generated-bindings/alp-double-query_multi_column-bindings.cu + generated-bindings/alp-float-decompress_column-bindings.cu + generated-bindings/alp-float-query_column-bindings.cu + generated-bindings/alp-float-query_multi_column-bindings.cu + generated-bindings/alpextended-double-decompress_column-bindings.cu + generated-bindings/alpextended-double-query_column-bindings.cu + generated-bindings/alpextended-double-query_multi_column-bindings.cu + generated-bindings/alpextended-float-decompress_column-bindings.cu + generated-bindings/alpextended-float-query_column-bindings.cu + generated-bindings/alpextended-float-query_multi_column-bindings.cu + generated-bindings/bp-uint32_t-decompress_column-bindings.cu + generated-bindings/bp-uint32_t-query_column-bindings.cu + generated-bindings/bp-uint64_t-decompress_column-bindings.cu + generated-bindings/bp-uint64_t-query_column-bindings.cu + generated-bindings/ffor-uint32_t-compute_column-bindings.cu + generated-bindings/ffor-uint32_t-decompress_column-bindings.cu + generated-bindings/ffor-uint32_t-query_column-bindings.cu + generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu + generated-bindings/ffor-uint64_t-compute_column-bindings.cu + generated-bindings/ffor-uint64_t-decompress_column-bindings.cu + generated-bindings/ffor-uint64_t-query_column-bindings.cu + generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu +) +add_library(generated-bindings STATIC ${GALP_GENERATED_BINDINGS_SOURCES}) +target_compile_options(generated-bindings PRIVATE $<$:-expt-relaxed-constexpr>) +target_include_directories(generated-bindings PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CUDAToolkit_INCLUDE_DIRS}) +target_link_libraries(generated-bindings PUBLIC CUDA::cudart nvcomp::nvcomp Thrust::Thrust galp_core) +apply_warnings(generated-bindings) +set_target_properties(generated-bindings PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + + +add_executable(compressors-benchmarks compressors-benchmarks.cu nvcomp/benchmark-compressors.cu nvcomp/nvcomp-compressors.cu) +apply_warnings(compressors-benchmarks) +target_link_libraries(compressors-benchmarks PRIVATE galp_core) + +add_executable(micro-benchmarks micro-benchmarks.cu nvcomp/benchmark-compressors.cu nvcomp/nvcomp-compressors.cu) +apply_warnings(micro-benchmarks) +target_link_libraries(micro-benchmarks PRIVATE galp_core generated-bindings) diff --git a/galp/benchmark/compressors-benchmarks.cu b/galp/benchmark/compressors-benchmarks.cu new file mode 100644 index 00000000..f40e7897 --- /dev/null +++ b/galp/benchmark/compressors-benchmarks.cu @@ -0,0 +1,248 @@ +// ============================================================================= +// benchmark_runner.cpp — single-file driver that mirrors test-scripts/run.sh +// +// * Builds with any C++17+ host compiler + NVCC. +// * Host-only code (filesystem, main, etc.) is hidden from the device pass +// via `#if !defined(__CUDA_ARCH__)` guards, so NVCC no longer chokes. +// +// Compile example: +// g++ -std=c++17 benchmark_runner.cpp -o compressors-benchmarks $(CUDA_LIBS) ... +// ============================================================================= +#include "engine/enums.cuh" +#include "flsgpu/consts.cuh" +#include "generator/generate_binaries.hpp" +#include "nvcomp/benchmark-compressors.cuh" +#include +#include +#include +#include +#if !defined(__CUDA_ARCH__) +#include // host-only +#endif +#include +#include +#include +#include +#include +#include + +// ----------------------------------------------------------------------------- +// Utility helpers (shared by host + device passes) +// ----------------------------------------------------------------------------- +inline std::string extract_filename(const std::string& path) { + const std::size_t pos = path.find_last_of('/'); + std::string fn = (pos == std::string::npos) ? path : path.substr(pos + 1); + const std::size_t ext = fn.find_last_of('.'); + return (ext == std::string::npos) ? fn : fn.substr(0, ext); +} + +struct CLIArgs { + enums::DataType data_type; + enums_nvcomp::ComparisonType comparison_type; + enums_nvcomp::CompressionType decompressor_enum; + std::string file_path; + int n_values; +}; + +inline CLIArgs parse_cli_args(int argc, char* argv[]) { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] + << " " + " \n"; + throw std::invalid_argument("Incorrect CLI argument count"); + } + + int idx = 0; + CLIArgs args; + args.data_type = enums::string_to_data_type(argv[++idx]); + args.comparison_type = enums_nvcomp::string_to_comparison_type(argv[++idx]); + args.decompressor_enum = enums_nvcomp::string_to_compression_type(argv[++idx]); + args.file_path = argv[++idx]; + args.n_values = std::atoi(argv[++idx]) * consts::VALUES_PER_VECTOR; + return args; +} + +namespace data::arrays { +template +std::pair read_file_as(const std::string path, const size_t input_count) { + // Open file + std::ifstream inputFile(path, std::ios::binary | std::ios::ate); + if (!inputFile) { + throw std::invalid_argument("Could not open the specified file."); + } + // Get file size + const std::streamsize file_size = inputFile.tellg(); + inputFile.seekg(0, std::ios::beg); + + // Check file size to contain right type of data + bool file_size_is_multiple_of_T_size = static_cast(file_size) % static_cast(sizeof(T)) != 0; + if (file_size_is_multiple_of_T_size) { + throw std::invalid_argument("File size is incorrect, it is not a multiple of the type's size."); + } + + const size_t values_in_file = static_cast(file_size) / sizeof(T); + size_t count = input_count == 0 ? values_in_file : input_count; + count = count - (count % consts::VALUES_PER_VECTOR); + auto column = new T[count]; + + // Read either the file size, or the total number of values needed, + // whichever is smaller + const std::streamsize read_size = std::min(file_size, static_cast((count * sizeof(T)))); + if (!inputFile.read(reinterpret_cast(column), read_size)) { + throw std::invalid_argument("Failed to read file into column"); + } + + inputFile.close(); + + // Copy paste the values in file until the column is filled + if (values_in_file < count) { + size_t n_filled_values = values_in_file; + size_t n_empty_values_column = count - n_filled_values; + while (n_empty_values_column != 0) { + size_t n_values_to_copy = std::min(n_empty_values_column, values_in_file); + std::memcpy(column + n_filled_values, column, n_values_to_copy * sizeof(T)); + n_filled_values += n_values_to_copy; + n_empty_values_column -= n_values_to_copy; + } + } + + return std::make_pair(column, count); +} +} // namespace data::arrays + +// Template benchmark launcher (host-only code but harmless for device pass) +template +inline void execute_benchmark(const CLIArgs& args) { + auto [data, count] = data::arrays::read_file_as(args.file_path, args.n_values); + + uint32_t u = 0x4100f9db; // deterministic test value + const T value_to_search_for = *reinterpret_cast(&u); + + BenchmarkResult result; + for (int i = 0; i < 2; ++i) { + if (args.comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY && + args.decompressor_enum == enums_nvcomp::THRUST) { + result = benchmark_thrust(data, count, value_to_search_for); + } else if (args.decompressor_enum == enums_nvcomp::ALP || args.decompressor_enum == enums_nvcomp::GALP) { + result = benchmark_alp(args.comparison_type, args.decompressor_enum, data, count, value_to_search_for); + } else { + result = benchmark_hwc(args.comparison_type, args.decompressor_enum, data, count, value_to_search_for); + } + } + + result.log_result( + args.comparison_type, args.decompressor_enum, count * sizeof(T), extract_filename(args.file_path)); + + delete data; +} + +// ----------------------------------------------------------------------------- +// Main driver (host-only; skipped entirely for the device pass) +// ----------------------------------------------------------------------------- +#if !defined(__CUDA_ARCH__) +int main(int argc, char* argv[]) { + constexpr int VECTOR_COUNT = 25'600; // matches shell-script constant + + // ------------------------------------------------------------------------- + // Single-case CLI fallback (exactly 5 user args) + // ------------------------------------------------------------------------- + if (argc == 6) { + CLIArgs args = parse_cli_args(argc, argv); + if (args.data_type == enums::DataType::F32) + execute_benchmark(args); + else if (args.data_type == enums::DataType::F64) + execute_benchmark(args); + else + throw std::invalid_argument("Unsupported data type"); + return 0; + } + + // ------------------------------------------------------------------------- + // (0) Generate new binaries + discover directories + // ------------------------------------------------------------------------- + const size_t TOTAL = 25'600 * 1'024; + const size_t HEAD = 0; + + const std::filesystem::path floats_dir = FLS_GALP_SOURCE_DIR "/data/floats"; + const std::filesystem::path doubles_dir = FLS_GALP_SOURCE_DIR "/data/doubles"; + + bin::GenResult gen = bin::generate_write_and_scan(floats_dir, doubles_dir, TOTAL, HEAD); + + std::cout << "First " << HEAD << " values:\n"; + for (double v : gen.head) + std::cout << v << '\n'; + + if (gen.float_files.empty() && gen.double_files.empty()) { + std::cerr << "No .bin test files found. Populate data/floats and data/doubles.\n"; + return 1; + } + + // ------------------------------------------------------------------------- + // Benchmark matrix settings (mirrors test-scripts/run.sh) + // ------------------------------------------------------------------------- + const std::vector COMPARISONS = {enums_nvcomp::ComparisonType::DECOMPRESSION, + enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY}; + + const std::vector COMPRESSORS = { + // + // enums_nvcomp::ALP, + enums_nvcomp::GALP, + enums_nvcomp::BITCOMP, + enums_nvcomp::BITCOMP_SPARSE, + enums_nvcomp::LZ4, + enums_nvcomp::ZSTD, + enums_nvcomp::DEFLATE, + enums_nvcomp::GDEFLATE, + enums_nvcomp::SNAPPY + // + }; + + // CSV header + std::printf("comparison_type,compression_type,data_name,found_value," + "n_bytes,execution_time_ms,compression_ratio\n"); + + // ------------------------------------------------------------------------- + // (1) Full matrix — Floats (F32) + // ------------------------------------------------------------------------- + for (auto cmp : COMPARISONS) + for (auto decomp : COMPRESSORS) + for (const auto& path : gen.float_files) { + CLIArgs args {enums::DataType::F32, cmp, decomp, path, VECTOR_COUNT * consts::VALUES_PER_VECTOR}; + execute_benchmark(args); + } + + // ------------------------------------------------------------------------- + // (2) Full matrix — Doubles (F64) + // ------------------------------------------------------------------------- + for (auto cmp : COMPARISONS) + for (auto decomp : COMPRESSORS) + for (const auto& path : gen.double_files) { + CLIArgs args {enums::DataType::F64, cmp, decomp, path, VECTOR_COUNT * consts::VALUES_PER_VECTOR}; + execute_benchmark(args); + } + + // ------------------------------------------------------------------------- + // (3) Thrust path — only DECOMPRESSION_QUERY + // ------------------------------------------------------------------------- + auto run_thrust = [&](enums::DataType dtype, const std::vector& files) { + for (const auto& path : files) { + CLIArgs args; + args.data_type = dtype; + args.comparison_type = enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY; + args.decompressor_enum = enums_nvcomp::THRUST; + args.file_path = path; + args.n_values = VECTOR_COUNT * consts::VALUES_PER_VECTOR; + + if (dtype == enums::DataType::F32) + execute_benchmark(args); + else + execute_benchmark(args); + } + }; + + run_thrust(enums::DataType::F32, gen.float_files); + run_thrust(enums::DataType::F64, gen.double_files); + + return 0; +} +#endif // !__CUDA_ARCH__ diff --git a/galp/benchmark/generated-bindings/alp-double-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alp-double-decompress_column-bindings.cu new file mode 100644 index 00000000..382d0a2d --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-double-decompress_column-bindings.cu @@ -0,0 +1,105 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +double* decompress_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-double-query_column-bindings.cu b/galp/benchmark/generated-bindings/alp-double-query_column-bindings.cu new file mode 100644 index 00000000..45690a79 --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-double-query_column-bindings.cu @@ -0,0 +1,106 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-double-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alp-double-query_multi_column-bindings.cu new file mode 100644 index 00000000..aa73e73a --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-double-query_multi_column-bindings.cu @@ -0,0 +1,50 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-float-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alp-float-decompress_column-bindings.cu new file mode 100644 index 00000000..83973c9b --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-float-decompress_column-bindings.cu @@ -0,0 +1,146 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +float* decompress_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-float-query_column-bindings.cu b/galp/benchmark/generated-bindings/alp-float-query_column-bindings.cu new file mode 100644 index 00000000..a815cbf8 --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-float-query_column-bindings.cu @@ -0,0 +1,147 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Dummy) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::DummyALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateless) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatelessALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::device::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alp-float-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alp-float-query_multi_column-bindings.cu new file mode 100644 index 00000000..8da7ec3d --- /dev/null +++ b/galp/benchmark/generated-bindings/alp-float-query_multi_column-bindings.cu @@ -0,0 +1,65 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Stateful) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>, + flsgpu::host::ALPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-double-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-double-decompress_column-bindings.cu new file mode 100644 index 00000000..318e9eb4 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-double-decompress_column-bindings.cu @@ -0,0 +1,134 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +double* decompress_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-double-query_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-double-query_column-bindings.cu new file mode 100644 index 00000000..17212ea4 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-double-query_column-bindings.cu @@ -0,0 +1,135 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-double-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-double-query_multi_column-bindings.cu new file mode 100644 index 00000000..f535dcd9 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-double-query_multi_column-bindings.cu @@ -0,0 +1,135 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>( + const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + double, + 1, + 1, + flsgpu::device::ALPDecompressor< + double, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + double, + 4, + 1, + flsgpu::device::ALPDecompressor< + double, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-float-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-float-decompress_column-bindings.cu new file mode 100644 index 00000000..da901b77 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-float-decompress_column-bindings.cu @@ -0,0 +1,189 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +float* decompress_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::decompress_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-float-query_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-float-query_column-bindings.cu new file mode 100644 index 00000000..2a841bfa --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-float-query_column-bindings.cu @@ -0,0 +1,190 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>( + const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::device::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/alpextended-float-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/alpextended-float-query_multi_column-bindings.cu new file mode 100644 index 00000000..fb160bb8 --- /dev/null +++ b/galp/benchmark/generated-bindings/alpextended-float-query_multi_column-bindings.cu @@ -0,0 +1,190 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>( + const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 32, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + float, + 1, + 1, + flsgpu::device::ALPDecompressor< + float, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::Naive) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::NaiveBranchless) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::NaiveBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAll) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::PrefetchAllBranchless) { + return kernels::host::query_multi_column< + float, + 4, + 1, + flsgpu::device::ALPDecompressor< + float, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllBranchlessALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>, + flsgpu::host::ALPExtendedColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column ALPExtended"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint32_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint32_t-decompress_column-bindings.cu new file mode 100644 index 00000000..b5e91439 --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint32_t-decompress_column-bindings.cu @@ -0,0 +1,85 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include +namespace bindings { + +template <> +uint32_t* +decompress_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 32, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint32_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint32_t-query_column-bindings.cu new file mode 100644 index 00000000..8188cd0c --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint32_t-query_column-bindings.cu @@ -0,0 +1,86 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 32, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint64_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint64_t-decompress_column-bindings.cu new file mode 100644 index 00000000..2b65bafe --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint64_t-decompress_column-bindings.cu @@ -0,0 +1,74 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +uint64_t* +decompress_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/bp-uint64_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/bp-uint64_t-query_column-bindings.cu new file mode 100644 index 00000000..c5bbe035 --- /dev/null +++ b/galp/benchmark/generated-bindings/bp-uint64_t-query_column-bindings.cu @@ -0,0 +1,74 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::BPDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::BPColumn>, + flsgpu::device::BPColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column BP"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-compute_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-compute_column-bindings.cu new file mode 100644 index 00000000..1429ca14 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-compute_column-bindings.cu @@ -0,0 +1,91 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool compute_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in compute_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-decompress_column-bindings.cu new file mode 100644 index 00000000..f80eb85a --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-decompress_column-bindings.cu @@ -0,0 +1,86 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +uint32_t* +decompress_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-query_column-bindings.cu new file mode 100644 index 00000000..93219940 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-query_column-bindings.cu @@ -0,0 +1,86 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu new file mode 100644 index 00000000..e3e6a4c1 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint32_t-query_multi_column-bindings.cu @@ -0,0 +1,60 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 32 && unpacker == enums::Unpacker::OldFls && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint32_t, + 1, + 32, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerOldFls>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint32_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint32_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint32_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-compute_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-compute_column-bindings.cu new file mode 100644 index 00000000..201d8b83 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-compute_column-bindings.cu @@ -0,0 +1,78 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool compute_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::compute_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn, + 10>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in compute_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-decompress_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-decompress_column-bindings.cu new file mode 100644 index 00000000..e44c1f8c --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-decompress_column-bindings.cu @@ -0,0 +1,74 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +uint64_t* +decompress_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::decompress_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, n_samples); + } + throw std::invalid_argument("Could not find correct binding in decompress_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-query_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-query_column-bindings.cu new file mode 100644 index 00000000..d1b237dc --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-query_column-bindings.cu @@ -0,0 +1,73 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include +namespace bindings { + +template <> +bool query_column>(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples) { + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::Dummy && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerDummy>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::device::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu b/galp/benchmark/generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu new file mode 100644 index 00000000..3ec5c427 --- /dev/null +++ b/galp/benchmark/generated-bindings/ffor-uint64_t-query_multi_column-bindings.cu @@ -0,0 +1,48 @@ + +#include "engine/kernels.cuh" +#include "engine/multi-column-host-kernels.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include + +namespace bindings { + +template <> +bool query_multi_column>(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples) { + + if (unpack_n_vectors == 1 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint64_t, + 1, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 1, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + + if (unpack_n_vectors == 4 && unpack_n_values == 1 && unpacker == enums::Unpacker::StatefulBranchless && + patcher == enums::Patcher::None) { + return kernels::host::query_multi_column< + uint64_t, + 4, + 1, + flsgpu::device::FFORDecompressor< + uint64_t, + 4, + flsgpu::device::BitUnpackerStatefulBranchless>, + flsgpu::device::FFORColumn>, + flsgpu::host::FFORColumn>(column, magic_value, n_samples); + } + throw std::invalid_argument("Could not find correct binding in query_multi_column FFOR"); +} + +} // namespace bindings diff --git a/galp/benchmark/include/generated-bindings/kernel-bindings.cuh b/galp/benchmark/include/generated-bindings/kernel-bindings.cuh new file mode 100644 index 00000000..4ba5b5f3 --- /dev/null +++ b/galp/benchmark/include/generated-bindings/kernel-bindings.cuh @@ -0,0 +1,211 @@ +#ifndef GENERATED_KERNEL_BINDINGS_CUH +#define GENERATED_KERNEL_BINDINGS_CUH + +#include "engine/enums.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include + +namespace bindings { + +template +T* decompress_column(const ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); + +template +bool query_column(const ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const T magic_value, + const uint32_t n_samples); + +template +bool compute_column(const ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples); + +template +bool query_multi_column(ColumnT column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const T magic_value, + const uint32_t n_samples); + +uint32_t* decompress_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +uint64_t* decompress_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +uint32_t* decompress_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +uint64_t* decompress_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +float* decompress_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +double* decompress_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +float* decompress_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); +double* decompress_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t n_samples); + +bool query_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::BPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_column(const flsgpu::device::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); + +bool compute_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples); +bool compute_column(const flsgpu::device::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const unsigned n_repetitions, + const uint32_t n_samples); + +bool query_multi_column(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint32_t magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::FFORColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const uint64_t magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const float magic_value, + const uint32_t n_samples); +bool query_multi_column(const flsgpu::host::ALPExtendedColumn column, + const unsigned unpack_n_vectors, + const unsigned unpack_n_values, + const enums::Unpacker unpacker, + const enums::Patcher patcher, + const double magic_value, + const uint32_t n_samples); + +} // namespace bindings + +#endif // GENERATED_KERNEL_BINDINGS_CUH diff --git a/galp/benchmark/include/generated-bindings/multi-column-device-kernels.cuh b/galp/benchmark/include/generated-bindings/multi-column-device-kernels.cuh new file mode 100644 index 00000000..6d1b8559 --- /dev/null +++ b/galp/benchmark/include/generated-bindings/multi-column-device-kernels.cuh @@ -0,0 +1,527 @@ +#ifndef MULTI_COLUMN_DEVICE_KERNELS_CUH +#define MULTI_COLUMN_DEVICE_KERNELS_CUH + +#include "engine/device-utils.cuh" +#include "flsgpu/flsgpu-api.cuh" + +namespace multi_column { + +template +__global__ void query_multi_column(const ColumnT column_0, const T value, bool* out) { + constexpr int32_t N_COLS = 1; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 1]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, const ColumnT column_1, const T value, bool* out) { + constexpr int32_t N_COLS = 2; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 2]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void +query_multi_column(const ColumnT column_0, const ColumnT column_1, const ColumnT column_2, const T value, bool* out) { + constexpr int32_t N_COLS = 3; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 3]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const T value, + bool* out) { + constexpr int32_t N_COLS = 4; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 4]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const T value, + bool* out) { + constexpr int32_t N_COLS = 5; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 5]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const T value, + bool* out) { + constexpr int32_t N_COLS = 6; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 6]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const T value, + bool* out) { + constexpr int32_t N_COLS = 7; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 7]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const ColumnT column_7, + const T value, + bool* out) { + constexpr int32_t N_COLS = 8; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 8]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + DecompressorT decompressor_7 = DecompressorT(column_7, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_7.unpack_next_into(registers + 7 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const ColumnT column_7, + const ColumnT column_8, + const T value, + bool* out) { + constexpr int32_t N_COLS = 9; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 9]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + DecompressorT decompressor_7 = DecompressorT(column_7, vector_index, lane); + DecompressorT decompressor_8 = DecompressorT(column_8, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_7.unpack_next_into(registers + 7 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_8.unpack_next_into(registers + 8 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +template +__global__ void query_multi_column(const ColumnT column_0, + const ColumnT column_1, + const ColumnT column_2, + const ColumnT column_3, + const ColumnT column_4, + const ColumnT column_5, + const ColumnT column_6, + const ColumnT column_7, + const ColumnT column_8, + const ColumnT column_9, + const T value, + bool* out) { + constexpr int32_t N_COLS = 10; + const auto mapping = VectorToWarpMapping(); + const int32_t vector_index = mapping.get_vector_index(); + const lane_t lane = mapping.get_lane(); + T registers[UNPACK_N_VALUES * UNPACK_N_VECS * 10]; + bool all_columns_equal = true; + DecompressorT decompressor_0 = DecompressorT(column_0, vector_index, lane); + DecompressorT decompressor_1 = DecompressorT(column_1, vector_index, lane); + DecompressorT decompressor_2 = DecompressorT(column_2, vector_index, lane); + DecompressorT decompressor_3 = DecompressorT(column_3, vector_index, lane); + DecompressorT decompressor_4 = DecompressorT(column_4, vector_index, lane); + DecompressorT decompressor_5 = DecompressorT(column_5, vector_index, lane); + DecompressorT decompressor_6 = DecompressorT(column_6, vector_index, lane); + DecompressorT decompressor_7 = DecompressorT(column_7, vector_index, lane); + DecompressorT decompressor_8 = DecompressorT(column_8, vector_index, lane); + DecompressorT decompressor_9 = DecompressorT(column_9, vector_index, lane); + for (si_t i {0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor_0.unpack_next_into(registers + 0 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_1.unpack_next_into(registers + 1 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_2.unpack_next_into(registers + 2 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_3.unpack_next_into(registers + 3 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_4.unpack_next_into(registers + 4 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_5.unpack_next_into(registers + 5 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_6.unpack_next_into(registers + 6 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_7.unpack_next_into(registers + 7 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_8.unpack_next_into(registers + 8 * (UNPACK_N_VALUES * UNPACK_N_VECS)); + decompressor_9.unpack_next_into(registers + 9 * (UNPACK_N_VALUES * UNPACK_N_VECS)); +#pragma unroll + for (int c {1}; c < N_COLS; ++c) { +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= + registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == + registers[va + v * UNPACK_N_VALUES + (c - 1) * UNPACK_N_VECS * UNPACK_N_VALUES]; + } + } + } +#pragma unroll + for (int va {0}; va < UNPACK_N_VALUES; ++va) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECS; ++v) { + all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value; + } + } + } + + if (all_columns_equal) { + *out = true; + } +} + +} // namespace multi_column +#endif // MULTI_COLUMN_DEVICE_KERNELS_CUH diff --git a/galp/benchmark/micro-benchmarks.cu b/galp/benchmark/micro-benchmarks.cu new file mode 100644 index 00000000..ccf7c656 --- /dev/null +++ b/galp/benchmark/micro-benchmarks.cu @@ -0,0 +1,261 @@ +#include "engine/data.cuh" +#include "engine/enums.cuh" +#include "engine/verification.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include "generated-bindings/kernel-bindings.cuh" +#include +#include +#include +#include +#include +#include +#include + +struct ProgramParameters { + enums::DataType data_type; + enums::Kernel kernel; + uint32_t unpack_n_vecs; + uint32_t unpack_n_vals; + enums::Unpacker unpacker; + enums::Patcher patcher; + data::ValueRange bit_width_range; + data::ValueRange ec_range; + size_t n_values; + uint32_t n_samples; + enums::Print print_option; +}; + +struct CLIArgs { + std::string data_type; + std::string kernel; + uint32_t unpack_n_vecs; + uint32_t unpack_n_vals; + std::string patcher; + std::string unpacker; + vbw_t start_vbw; + vbw_t end_vbw; + uint16_t start_ec; + uint16_t end_ec; + size_t n_vecs; + uint32_t n_samples; + uint32_t print_debug; + + CLIArgs(const int argc, char** argv) { + constexpr int32_t CORRECT_ARG_COUNT = 14; + if (argc != CORRECT_ARG_COUNT) { + throw std::invalid_argument("Wrong arg count.\n"); + } + + int32_t argcounter = 0; + data_type = argv[++argcounter]; + kernel = argv[++argcounter]; + unpack_n_vecs = std::stoul(argv[++argcounter]); + unpack_n_vals = std::stoul(argv[++argcounter]); + unpacker = argv[++argcounter]; + patcher = argv[++argcounter]; + start_vbw = std::stoul(argv[++argcounter]); + end_vbw = std::stoul(argv[++argcounter]); + start_ec = std::stoul(argv[++argcounter]); + end_ec = std::stoul(argv[++argcounter]); + n_vecs = std::stoul(argv[++argcounter]); + n_samples = std::stoul(argv[++argcounter]); + print_debug = std::stoul(argv[++argcounter]); + } + + ProgramParameters parse() { + return ProgramParameters { + enums::string_to_data_type(data_type), + enums::string_to_kernel(kernel), + unpack_n_vecs, + unpack_n_vals, + enums::string_to_unpacker(unpacker), + enums::string_to_patcher(patcher), + data::ValueRange(start_vbw, end_vbw), + data::ValueRange(start_ec, end_ec), + n_vecs * consts::VALUES_PER_VECTOR, + n_samples, + static_cast(print_debug), + }; + } + +private: +}; + +template +verification::ExecutionResult decompress_column(const ColumnT column, const ProgramParameters params) { + auto column_device = column.copy_to_device(); + const T* out = bindings::decompress_column( + column_device, params.unpack_n_vecs, params.unpack_n_vals, params.unpacker, params.patcher, params.n_samples); + flsgpu::host::free_column(column_device); + + const T* correct_out = data::bindings::decompress(column); + auto result = verification::compare_data(correct_out, out, params.n_values); + delete correct_out; + delete out; + return result; +} + +template +verification::ExecutionResult +query_column(const ColumnT column, const ProgramParameters params, const bool query_result, const T magic_value) { + auto column_device = column.copy_to_device(); + const bool answer = bindings::query_column(column_device, + params.unpack_n_vecs, + params.unpack_n_vals, + params.unpacker, + params.patcher, + magic_value, + params.n_samples); + flsgpu::host::free_column(column_device); + + // Weird hack to avoid refactor_ + T a = query_result ? 1.0 : 0.0; + T b = answer ? 1.0 : 0.0; + + return verification::compare_data(&a, &b, 1); +} + +template +verification::ExecutionResult +query_multi_column(const ColumnT column, const ProgramParameters params, const bool query_result, const T magic_value) { + const bool answer = bindings::query_multi_column(column, + params.unpack_n_vecs, + params.unpack_n_vals, + params.unpacker, + params.patcher, + magic_value, + params.n_samples); + + // Weird hack to avoid refactor_ + T a = query_result ? 1.0 : 0.0; + T b = answer ? 1.0 : 0.0; + + return verification::compare_data(&a, &b, 1); +} + +template +verification::ExecutionResult +execute_kernel(const ColumnT column, const ProgramParameters params, const bool query_result, const T magic_value) { + if (params.kernel == enums::Kernel::Decompress) { + return decompress_column(column, params); + } else if (params.kernel == enums::Kernel::Query) { + return query_column(column, params, query_result, magic_value); + } else if (params.kernel == enums::Kernel::QueryMultiColumn) { + return query_multi_column(column, params, query_result, magic_value); + } else { + throw std::invalid_argument("Kernel not implemented yet.\n"); + } +} + +template +std::vector> execute_ffor(const ProgramParameters params) { + using UINT_T = typename utils::same_width_uint::type; + auto results = std::vector>(); + + for (vbw_t vbw {params.bit_width_range.min}; vbw <= params.bit_width_range.max; ++vbw) { + auto vbw_range = data::ValueRange(vbw); + if (params.kernel == enums::Kernel::QueryMultiColumn) { + vbw_range = params.bit_width_range; + } + bool query_result = false; + T magic_value = consts::as::MAGIC_NUMBER; + flsgpu::host::FFORColumn column; + + if (params.kernel == enums::Kernel::Query) { + auto [_query_result, _column] = + data::columns::generate_binary_ffor_column(params.n_values, vbw_range, params.unpack_n_vecs); + query_result = _query_result; + column = _column; + } else { + column = data::columns::generate_random_ffor_column( + params.n_values, vbw_range, data::ValueRange(0, 100), params.unpack_n_vecs); + } + + if (params.kernel == enums::Kernel::QueryMultiColumn) { + // We do not want query multicolumn to ever find a full lane of the + // value to query to limit write bandwidth + magic_value = std::numeric_limits::max(); + } + + results.push_back(execute_kernel>(column, params, query_result, magic_value)); + + flsgpu::host::free_column(column); + + if (params.kernel == enums::Kernel::QueryMultiColumn) { + break; + } + } + + return results; +} + +template +std::vector> execute_alp(const ProgramParameters params) { + using UINT_T = typename utils::same_width_uint::type; + auto results = std::vector>(); + + // for (vbw_t vbw{params.bit_width_range.min}; vbw <= + // params.bit_width_range.max; ++vbw) { + { + bool query_result = false; + T magic_value = consts::as::MAGIC_NUMBER; + + auto column = data::columns::generate_alp_column( + params.n_values, params.bit_width_range, data::ValueRange(0), params.unpack_n_vecs); + for (uint16_t ec {params.ec_range.min}; ec <= params.ec_range.max; ++ec) { + column = data::columns::modify_alp_exception_count(column, ec); + + if (params.kernel == enums::Kernel::Query) { + auto [_query_result, _magic_value] = + data::columns::get_value_to_query>(column); + query_result = _query_result; + magic_value = _magic_value; + } + + if (params.patcher == enums::Patcher::Dummy || params.patcher == enums::Patcher::Stateless || + params.patcher == enums::Patcher::Stateful) { + results.push_back( + execute_kernel>(column, params, query_result, magic_value)); + } else { + auto column_extended = column.create_extended_column(); + + results.push_back(execute_kernel>( + column_extended, params, query_result, magic_value)); + + flsgpu::host::free_column(column_extended); + } + } + + flsgpu::host::free_column(column); + } + + return results; +} + +int main(int argc, char** argv) { + CLIArgs args(argc, argv); + ProgramParameters params = args.parse(); + + int32_t exit_code = 0; + bool print_debug = params.print_option != enums::Print::PrintNothing; + switch (params.data_type) { + case enums::DataType::U32: + exit_code = verification::process_results(execute_ffor(params), print_debug); + break; + case enums::DataType::U64: + exit_code = verification::process_results(execute_ffor(params), print_debug); + break; + case enums::DataType::F32: + exit_code = verification::process_results(execute_alp(params), print_debug); + break; + case enums::DataType::F64: + exit_code = verification::process_results(execute_alp(params), print_debug); + break; + } + + if (params.print_option == enums::Print::PrintDebugExit0) { + exit(0); + } + + exit(exit_code); +} diff --git a/galp/benchmark/nvcomp/benchmark-compressors.cu b/galp/benchmark/nvcomp/benchmark-compressors.cu new file mode 100644 index 00000000..cb6623f0 --- /dev/null +++ b/galp/benchmark/nvcomp/benchmark-compressors.cu @@ -0,0 +1,211 @@ +#include "alp/alp-bindings.cuh" +#include "engine/data.cuh" +#include "engine/device-utils.cuh" +#include "engine/kernels.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include "nvcomp/benchmark-compressors.cuh" +#include "nvcomp/nvcomp-compressors.cuh" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template +using ALPDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>; + +template +using ALPExtendedDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>; + +template +BenchmarkResult benchmark_thrust(const T* input, const size_t value_count, const T value_to_search_for) { + CudaStopwatch stopwatch = CudaStopwatch(); + double execution_time_ms; + + thrust::device_vector d_vec(input, input + value_count); + stopwatch.start(); + bool result = thrust::any_of(thrust::device, d_vec.begin(), d_vec.end(), is_equal_to(value_to_search_for)); + execution_time_ms = stopwatch.stop(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + + return BenchmarkResult {result, execution_time_ms, 1.0}; +} + +template +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const T* input, + const flsgpu::host::ALPColumn column, + const T value_to_search_for) { + bool result = false; + GPUArray d_query_result(1, &result); + GPUArray d_decompression_result(column.get_n_values()); + + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + CudaStopwatch stopwatch = CudaStopwatch(); + double execution_time_ms; + double compression_ratio; + + switch (decompressor_enum) { + case enums_nvcomp::ALP: { + flsgpu::device::ALPColumn d_column = column.copy_to_device(); + stopwatch.start(); + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + kernels::device::decompress_column, + flsgpu::device::ALPColumn> + <<>>(d_column, d_decompression_result.get()); + } else { + kernels::device:: + query_column, flsgpu::device::ALPColumn> + <<>>( + d_column, d_query_result.get(), value_to_search_for); + } + execution_time_ms = stopwatch.stop(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + flsgpu::host::free_column(d_column); + compression_ratio = column.get_compression_ratio(); + } break; + + case enums_nvcomp::GALP: { + flsgpu::host::ALPExtendedColumn column_extended = column.create_extended_column(); + flsgpu::device::ALPExtendedColumn d_column = column_extended.copy_to_device(); + stopwatch.start(); + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + kernels::device::decompress_column, + flsgpu::device::ALPExtendedColumn> + <<>>(d_column, d_decompression_result.get()); + } else { + kernels::device::query_column, + flsgpu::device::ALPExtendedColumn> + <<>>( + d_column, d_query_result.get(), value_to_search_for); + } + execution_time_ms = stopwatch.stop(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + compression_ratio = column_extended.get_compression_ratio(); + flsgpu::host::free_column(column_extended); + flsgpu::host::free_column(d_column); + } break; + default: + throw std::invalid_argument("Could not parse decompressor enum for alp\n"); + } + + bool kernel_successful = false; + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + GPUArray d_input(column.get_n_values(), input); + + kernel_successful = + check_if_device_buffers_are_equal(d_decompression_result.get(), d_input.get(), column.get_n_values()); + } + + return BenchmarkResult {kernel_successful, execution_time_ms, compression_ratio}; +} + +template +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const T* input, + const size_t value_count, + const T value_to_search_for) { + flsgpu::host::ALPColumn column = alp::encode(input, value_count, true); + auto result = benchmark_alp(comparison_type, decompressor_enum, input, column, value_to_search_for); + flsgpu::host::free_column(column); + + return result; +} + +template +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const T* input, + const size_t value_count, + const T value_to_search_for) { + size_t size_in_bytes = value_count * sizeof(T); + GPUArray d_input_buffer(size_in_bytes, reinterpret_cast(input)); + hwc::Compressor compressor(compression_type); + hwc::CompressedBuffer d_compressed_buffer = compressor.compress(d_input_buffer.get(), size_in_bytes); + + GPUArray d_query_result(1); + double compression_ratio = d_compressed_buffer.get_compression_ratio(); + + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, utils::get_n_vecs_from_size(value_count)); + + CudaStopwatch stopwatch = CudaStopwatch(); + uint8_t* d_output_buffer = compressor.decompress(d_compressed_buffer, stopwatch); + + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY) { + hwc::DummyColumn d_column {reinterpret_cast(d_output_buffer), value_count}; + kernels::device::query_column, hwc::DummyColumn> + <<>>(d_column, d_query_result.get(), value_to_search_for); + } + + double execution_time_ms = stopwatch.get_result(); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + + bool kernel_successful = false; + if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION) { + kernel_successful = + check_if_device_buffers_are_equal(d_output_buffer, d_input_buffer.get(), size_in_bytes); + } else if (comparison_type == enums_nvcomp::ComparisonType::DECOMPRESSION_QUERY) { + d_query_result.copy_to_host(&kernel_successful); + } + + CUDA_SAFE_CALL(cudaFree(d_output_buffer)); + d_compressed_buffer.free(); + compressor.free(); + + return BenchmarkResult {kernel_successful, execution_time_ms, compression_ratio}; +} + +template BenchmarkResult +benchmark_thrust(const float* input, const size_t value_count, const float value_to_search_for); +template BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const float* input, + const size_t value_count, + const float value_to_search_for); +template BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const float* input, + const size_t value_count, + const float value_to_search_for); + +template BenchmarkResult +benchmark_thrust(const double* input, const size_t value_count, const double value_to_search_for); +template BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const double* input, + const size_t value_count, + const double value_to_search_for); +template BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const double* input, + const size_t value_count, + const double value_to_search_for); diff --git a/galp/benchmark/nvcomp/nvcomp-compressors.cu b/galp/benchmark/nvcomp/nvcomp-compressors.cu new file mode 100644 index 00000000..f21a14fd --- /dev/null +++ b/galp/benchmark/nvcomp/nvcomp-compressors.cu @@ -0,0 +1,80 @@ +#include "nvcomp/nvcomp-compressors.cuh" +#include +#include +#include +#include +#include + +namespace hwc { +nvcompCompressionManager* get_compressor_manager(const enums_nvcomp::CompressionType compression_type, + const nvcompType_t data_type, + const size_t chunk_size) { + switch (compression_type) { + case enums_nvcomp::CompressionType::BITCOMP: { + nvcompBatchedBitcompOpts_t format_opts {0, data_type}; + return new nvcomp::BitcompManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::BITCOMP_SPARSE: { + nvcompBatchedBitcompOpts_t format_opts {1, data_type}; + return new nvcomp::BitcompManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::LZ4: { + nvcompBatchedLZ4Opts_t format_opts {data_type}; + return new nvcomp::LZ4Manager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::ZSTD: { + nvcompBatchedZstdOpts_t format_opts {data_type}; + return new nvcomp::ZstdManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::DEFLATE: { + nvcompBatchedDeflateOpts_t format_opts {data_type}; + return new nvcomp::DeflateManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::GDEFLATE: { + nvcompBatchedGdeflateOpts_t format_opts {data_type}; + return new nvcomp::GdeflateManager {chunk_size, format_opts}; + } break; + case enums_nvcomp::CompressionType::SNAPPY: { + nvcompBatchedSnappyOpts_t format_opts {data_type}; + return new nvcomp::SnappyManager {chunk_size, format_opts}; + } break; + default: + throw std::invalid_argument("Compression type is not supported"); + } +} + +__global__ void +check_buffer_equality(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length, bool* out) { + constexpr int32_t N_ELEMENTS_PER_THREAD = 4 * 32; + size_t index = N_ELEMENTS_PER_THREAD * (threadIdx.x + blockIdx.x * blockDim.x); + + bool buffers_are_equal = true; + +#pragma unroll + for (int32_t i {0}; i < N_ELEMENTS_PER_THREAD; ++i) { + if (index + i < length) { + buffers_are_equal &= buffer_a[index + i] == buffer_b[index + i]; + } + } + + if (!buffers_are_equal) { + *out = false; + } +} + +bool compare_d_buffers(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length) { + bool result = true; + bool* d_result; + CUDA_SAFE_CALL(cudaMalloc(&d_result, sizeof(bool) * 1)); + CUDA_SAFE_CALL(cudaMemcpy(d_result, &result, sizeof(bool) * 1, cudaMemcpyHostToDevice)); + + const int n_blocks = (length + (1024 * 128) - 1) / (1024 * 128); + check_buffer_equality<<>>(buffer_a, buffer_b, length, d_result); + + CUDA_SAFE_CALL(cudaMemcpy(&result, d_result, sizeof(bool) * 1, cudaMemcpyDeviceToHost)); + CUDA_SAFE_CALL(cudaFree(d_result)); + + return result; +} + +} // namespace hwc diff --git a/galp/code-generators/generate-kernel-bindings.py b/galp/code-generators/generate-kernel-bindings.py new file mode 100755 index 00000000..6ebd142f --- /dev/null +++ b/galp/code-generators/generate-kernel-bindings.py @@ -0,0 +1,429 @@ +#!/usr/bin/python3 + +import os +import sys + +import argparse +import logging + +GENERATED_BINDINGS_DIR = "./src/generated-bindings/" + +FILE_HEADER = """ +#include + +#include "kernel-bindings.cuh" +#include "../engine/kernels.cuh" +#include "../engine/multi-column-host-kernels.cuh" + +namespace bindings{ +""" + +FILE_FOOTER = """ +} +""" + + +DATA_TYPES = [ + "uint32_t", + "uint64_t", + "float", + "double", +] + +FUNCTIONS = [ + "decompress_column", + "query_column", + "compute_column", + "query_multi_column", +] + +ENCODINGS = [ + "BP", + "FFOR", + "ALP", + "ALPExtended", +] + +UNPACKERS = [ + "Dummy", + "OldFls", + #"SwitchCase", + #"Stateless", + #"StatelessBranchless", + #"StatefulCache", + #"StatefulLocal1", + #"StatefulLocal2", + #"StatefulLocal4", + #"StatefulShared1", + #"StatefulShared2", + #"StatefulShared4", + #"StatefulRegister1", + #"StatefulRegister2", + #"StatefulRegister4", + #"StatefulRegisterBranchless1", + #"StatefulRegisterBranchless2", + #"StatefulRegisterBranchless4", + "StatefulBranchless", +] +MULTI_COLUMN_UNPACKERS = [ + UNPACKERS[1], + UNPACKERS[2], +] + +PATCHERS = [ + "None", + "Dummy", + "Stateless", + "Stateful", + "Naive", + "NaiveBranchless", + "PrefetchAll", + "PrefetchAllBranchless", +] +MULTI_COLUMN_PATCHERS = [ + PATCHERS[0], + PATCHERS[3], + PATCHERS[4], + PATCHERS[5], + PATCHERS[6], + PATCHERS[7], +] + + +def get_column_t( + encoding: str, data_type: str, function: str, for_decompressor: bool = False +) -> str: + column_t = f"BPColumn<{data_type}>" + if "FFOR" in encoding: + column_t = f"FFORColumn<{data_type}>" + elif "ALPExtended" in encoding: + column_t = f"ALPExtendedColumn<{data_type}>" + elif "ALP" in encoding: + column_t = f"ALPColumn<{data_type}>" + return ( + "flsgpu::device::" + if function != "query_multi_column" or for_decompressor + else "flsgpu::host::" + ) + column_t + + +def get_decompressor_type( + encoding: str, + data_type: str, + function: str, + unpacker: str, + patcher: str, + n_vec: int, + n_val: int, +) -> str: + column_t = get_column_t(encoding, data_type, function, True) + functor = f"BPFunctor<{data_type}>" + patcher_t = f"" + decompressor_t = f"{encoding}Decompressor" + if "FFOR" in encoding: + functor = f"FFORFunctor<{data_type}, {n_vec}>" + elif "ALPExtended" in encoding: + functor = f"ALPFunctor<{data_type}, {n_vec}>" + patcher_t = f"flsgpu::device::{patcher}ALPExceptionPatcher<{data_type}, {n_vec}, {n_val}>," + decompressor_t = "ALPDecompressor" + elif "ALP" in encoding: + functor = f"ALPFunctor<{data_type}, {n_vec}>" + patcher_t = f"flsgpu::device::{patcher}ALPExceptionPatcher<{data_type}, {n_vec}, {n_val}>," + + loader_t = "" + if "Stateful" in unpacker and "StatefulBranchless" not in unpacker: + loader_t = ", flsgpu::device::" + if "Cache" in unpacker: + loader_t += f"CacheLoader<{data_type}, {n_vec}>" + elif "Local" in unpacker: + loader_t += f"LocalMemoryLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + elif "Shared" in unpacker: + loader_t += f"SharedMemoryLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + elif "RegisterBranchless" in unpacker: + loader_t += ( + f"RegisterBranchlessLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + ) + elif "Register" in unpacker: + loader_t += f"RegisterLoader<{data_type}, {n_vec}, {unpacker[-1]}>" + unpacker = "Stateful" + + unpacker_t = f"flsgpu::device::BitUnpacker{unpacker}<{data_type}, {n_vec}, {n_val}, flsgpu::device::{functor} {loader_t}>" + + return f"flsgpu::device::{decompressor_t}<{data_type}, {n_vec}, {unpacker_t}, {patcher_t} {column_t}>" + + +def get_if_statement( + encoding: str, + data_type: str, + function: str, + n_vec: int, + n_val: int, + unpacker: str, + patcher: str, + is_query_column: bool = False, + n_columns: int | None = None, + n_repetitions: int | None = None, +) -> str: + assert data_type in DATA_TYPES + assert function in FUNCTIONS + assert n_vec in [1, 2, 4, 8] + assert n_val in [1, 32] + assert unpacker in UNPACKERS + assert patcher in PATCHERS + + column_t = get_column_t(encoding, data_type, function) + decompressor_t = get_decompressor_type( + encoding, data_type, function, unpacker, patcher, n_vec, n_val + ) + extra_param = ( + "," + str(n_columns) + if n_columns + else ", magic_value" if is_query_column else "" + ) + return ( + f"if (unpack_n_vectors == {n_vec} && unpack_n_values == {n_val} && unpacker == enums::Unpacker::{unpacker} && patcher == enums::Patcher::{patcher} {'&& n_columns == ' + str(n_columns) if n_columns else ''}) " + + "{" # } + f"return kernels::host::{function}<{data_type}, {n_vec}, {n_val}, {decompressor_t}, {column_t} {',' + str(n_repetitions) if n_repetitions else ''}>(column {extra_param}, n_samples);" + "}" + ) + + +def get_function( + encoding: str, + data_type: str, + function: str, + return_type: str, + content: list[str], + is_query_column: bool = False, + is_multi_column: bool = False, + is_compute_column: bool = False, +) -> str: + assert not (is_multi_column and is_compute_column) + column_t = get_column_t(encoding, data_type, function) + return ( + f"template<> {return_type} {function}<{data_type},{column_t}>(const {column_t} column, const unsigned unpack_n_vectors, const unsigned unpack_n_values, const enums::Unpacker unpacker, const enums::Patcher patcher {', const ' + data_type + ' magic_value' if is_query_column or is_multi_column else ''}{', const unsigned n_repetitions' if is_compute_column else ''}, const uint32_t n_samples)" + + "{" + + "\n".join(content) + + f'throw std::invalid_argument("Could not find correct binding in {function} {encoding}<{data_type}>");' + + "}" + ) + + +def write_file( + file_name: str, + functions: list[str], +): + logging.info(f"Writing file {file_name}") + with open(os.path.join(GENERATED_BINDINGS_DIR, file_name), "w") as f: + f.write("\n".join([FILE_HEADER] + functions + [FILE_FOOTER])) + + +def get_if_statement_check_wrapper( + disable_unnecessary: bool, + encoding: str, + data_type: str, + function: str, + n_vec: int, + n_val: int, + unpacker: str, + patcher: str, + is_query_column: bool = False, + n_columns: int | None = None, + n_repetitions: int | None = None, +) -> str: + # du handling + is_necessary = ( + data_type == "float" + and function == "decompress_column" + and unpacker == "OldFls" + ) + + # OldFls handling + if unpacker == "OldFls": + n_val = 32 + + # Filters + unnessary_filter = disable_unnecessary and not is_necessary + switch_case_filter = unpacker == "SwitchCase" and ( + n_vec != 1 + or n_val != 1 + or "uint" not in data_type + or function == "compute_column" + ) + multi_column_filter = function == "query_multi_column" and ( + unpacker not in MULTI_COLUMN_UNPACKERS + or patcher not in MULTI_COLUMN_PATCHERS + or args.disable_multi_column + ) + old_fls_filter = unpacker == "OldFls" and ( + n_vec != 1 or data_type not in ["uint32_t", "float"] + ) + + is_filtered = ( + unnessary_filter or switch_case_filter or multi_column_filter or old_fls_filter + ) + if is_filtered: + return "" + + return get_if_statement( + encoding, + data_type, + function, + n_vec, + n_val, + unpacker, + patcher, + is_query_column, + n_columns, + n_repetitions, + ) + + +def main(args): + for encoding in ["BP", "FFOR"]: + for data_type in ["uint32_t", "uint64_t"]: + for binding, is_query_column in zip( + ["decompress_column", "query_column"], [False, True] + ): + write_file( + f"{encoding.lower()}-{data_type}-{binding}-bindings.cu", + [ + get_function( + encoding, + data_type, + binding, + "bool" if is_query_column else data_type + "*", + [ + get_if_statement_check_wrapper( + args.disable_unnecessary, + encoding, + data_type, + binding, + n_vec, + n_val, + unpacker, + "None", + is_query_column=is_query_column, + ) + for n_vec in [1, 4] + for n_val in [1] + for unpacker in UNPACKERS + ], + is_query_column=is_query_column, + ) + ], + ) + + for encoding in ["FFOR"]: + for data_type in ["uint32_t", "uint64_t"]: + for binding in ["query_multi_column", "compute_column"]: + is_compute_column = binding == "compute_column" + is_multi_column = binding == "query_multi_column" + write_file( + f"{encoding.lower()}-{data_type}-{binding}-bindings.cu", + [ + get_function( + encoding, + data_type, + binding, + "bool", + [ + get_if_statement_check_wrapper( + args.disable_unnecessary, + encoding, + data_type, + binding, + n_vec, + n_val, + unpacker, + "None", + n_repetitions=10 if is_compute_column else None, + is_query_column=is_multi_column, + ) + for n_vec in [1, 4] + for n_val in [1] + for unpacker in UNPACKERS + ], + is_multi_column=is_multi_column, + is_compute_column=is_compute_column, + ) + ], + ) + + for encoding, patchers_per_encoding in zip( + ["ALP", "ALPExtended"], [PATCHERS[1:4], PATCHERS[4:]] + ): + for data_type in ["float", "double"]: + for binding in ["decompress_column", "query_column", "query_multi_column"]: + is_query_column = binding == "query_column" + is_multi_column = binding == "query_multi_column" + write_file( + f"{encoding.lower()}-{data_type}-{binding}-bindings.cu", + [ + get_function( + encoding, + data_type, + binding, + ( + "bool" + if is_query_column or is_multi_column + else data_type + "*" + ), + [ + get_if_statement_check_wrapper( + args.disable_unnecessary, + encoding, + data_type, + binding, + n_vec, + n_val, + unpacker, + patcher, + is_query_column=is_query_column or is_multi_column, + n_repetitions=None, + ) + for n_vec in [1, 4] + for n_val in [1] + for unpacker in UNPACKERS[1:] + for patcher in patchers_per_encoding + ], + is_query_column=is_query_column, + is_multi_column=is_multi_column, + ) + ], + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog="program") + + parser.add_argument( + "-du", + "--disable-unnecessary", + type=bool, + default=False, + action=argparse.BooleanOptionalAction, + ) + parser.add_argument( + "-dmc", + "--disable-multi-column", + type=bool, + default=False, + action=argparse.BooleanOptionalAction, + ) + parser.add_argument( + "-ll", + "--logging-level", + type=int, + default=logging.INFO, + choices=[logging.CRITICAL, logging.ERROR, logging.INFO, logging.DEBUG], + help=f"logging level to use: {logging.CRITICAL}=CRITICAL, {logging.ERROR}=ERROR, {logging.INFO}=INFO, " + + f"{logging.DEBUG}=DEBUG, higher number means less output", + ) + + args = parser.parse_args() + logging.basicConfig(level=args.logging_level) # filename='program.log', + logging.info( + f"Started {os.path.basename(sys.argv[0])} with the following args: {args}" + ) + main(args) diff --git a/galp/code-generators/generate-multicolumn-kernels.py b/galp/code-generators/generate-multicolumn-kernels.py new file mode 100755 index 00000000..b60f97a1 --- /dev/null +++ b/galp/code-generators/generate-multicolumn-kernels.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +import os +import sys + +import argparse +import logging + +FILE_HEADER = """ +#include "../flsgpu/flsgpu-api.cuh" +#include "../engine/device-utils.cuh" + +#ifndef MULTI_COLUMN_DEVICE_KERNELS_CUH +#define MULTI_COLUMN_DEVICE_KERNELS_CUH + +namespace multi_column { +""" + +FILE_FOOTER = """ +} +#endif // MULTI_COLUMN_DEVICE_KERNELS_CUH +""" + +def generate_global_function(n_cols: int): + col_range = range(n_cols) + return "\n".join( + [ + "template ", + "__global__ void query_multi_column(" + + ",".join([f"const ColumnT column_{i}" for i in col_range]) + + ", const T value, bool *out) {", + f"constexpr int32_t N_COLS = {n_cols};", + "const auto mapping = VectorToWarpMapping();", + "const int32_t vector_index = mapping.get_vector_index();", + "const lane_t lane = mapping.get_lane();", + f"T registers[UNPACK_N_VALUES * UNPACK_N_VECS * {n_cols}];", + "bool all_columns_equal = true;", + "\n".join( + [ + f"DecompressorT decompressor_{i} = DecompressorT(column_{i}, vector_index, lane);" + for i in col_range + ] + ), + "for (si_t i{0}; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) {", + "\n".join( + [ + f"decompressor_{i}.unpack_next_into(registers + {i} * (UNPACK_N_VALUES * UNPACK_N_VECS));" + for i in col_range + ] + ), + "#pragma unroll", + "for (int c{1}; c < N_COLS; ++c) {", + "#pragma unroll", + "for (int va{0}; va < UNPACK_N_VALUES; ++va) {", + "#pragma unroll", + "for (int v{0}; v < UNPACK_N_VECS; ++v) {", + "all_columns_equal &= registers[va + v * UNPACK_N_VALUES + c * UNPACK_N_VECS * UNPACK_N_VALUES] == registers[va + v * UNPACK_N_VALUES + (c-1) * UNPACK_N_VECS * UNPACK_N_VALUES];", + "}", + "}", + "}", + "#pragma unroll", + "for (int va{0}; va < UNPACK_N_VALUES; ++va) {", + "#pragma unroll", + "for (int v{0}; v < UNPACK_N_VECS; ++v) {", + "all_columns_equal &= registers[va + v * UNPACK_N_VALUES] == value;", + "}", + "}", + "}", + "", + "if (all_columns_equal) {", + "*out = true;", + "}}", + ] + ) + + +def main(args): + n_cols = 10 + generated_str = "\n\n".join( + [ + FILE_HEADER, + *[generate_global_function(n) for n in range(1, n_cols + 1)], + FILE_FOOTER, + ] + ) + + with open("src/generated-bindings/multi-column-device-kernels.cuh", "w") as file: + file.write(generated_str) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog="program") + + parser.add_argument( + "-ll", + "--logging-level", + type=int, + default=logging.INFO, + choices=[logging.CRITICAL, logging.ERROR, logging.INFO, logging.DEBUG], + help=f"logging level to use: {logging.CRITICAL}=CRITICAL, {logging.ERROR}=ERROR, {logging.INFO}=INFO, " + + f"{logging.DEBUG}=DEBUG, higher number means less output", + ) + + args = parser.parse_args() + logging.basicConfig(level=args.logging_level) # filename='program.log', + logging.info( + f"Started {os.path.basename(sys.argv[0])} with the following args: {args}" + ) + main(args) diff --git a/galp/src/CMakeLists.txt b/galp/src/CMakeLists.txt new file mode 100644 index 00000000..9592f512 --- /dev/null +++ b/galp/src/CMakeLists.txt @@ -0,0 +1,12 @@ +set(GALP_CORE_SOURCES + alp/alp-bindings.cu + engine/enums.cu +) +add_library(galp_core STATIC ${GALP_CORE_SOURCES}) +target_compile_options(galp_core PRIVATE $<$:-expt-relaxed-constexpr>) +target_include_directories(galp_core PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CUDAToolkit_INCLUDE_DIRS}) +target_link_libraries(galp_core PUBLIC CUDA::cudart nvcomp::nvcomp Thrust::Thrust FastLanes::core) +apply_warnings(galp_core) +set_target_properties(galp_core PROPERTIES CUDA_SEPARABLE_COMPILATION ON) \ No newline at end of file diff --git a/galp/src/alp/alp-bindings.cu b/galp/src/alp/alp-bindings.cu new file mode 100644 index 00000000..72b02a59 --- /dev/null +++ b/galp/src/alp/alp-bindings.cu @@ -0,0 +1,263 @@ +#include "alp.hpp" +#include "alp/alp-bindings.cuh" +#include +#include +#include +#include +#include +#include + +namespace alp { +constexpr int MAX_ATTEMPTS_TO_ENCODE = 10000; + +template +size_t get_bytes_overhead_size_per_alp_vector() { + return sizeof(uint8_t) + // bit_width + sizeof(uint8_t) + // factor-idx + sizeof(uint8_t) + // exponent-idx + sizeof(T); // ffor base + // + 32; // Overhead of vector offset in packed array + // + 32; // Overhead of vector offset in exception array +}; + +template +size_t get_bytes_overhead_size_per_alp_extended_vector() { + return get_bytes_overhead_size_per_alp_vector() + + sizeof(uint16_t) * utils::get_n_lanes(); // pos + offset per lane + // data parallel_format; +}; + +template +size_t get_bytes_vector_compressed_size_without_overhead(const uint8_t bit_width, const uint16_t exceptions_count) { + constexpr size_t line_size = utils::get_n_lanes() * sizeof(T); + constexpr size_t exception_value_size = sizeof(T); + constexpr size_t exception_position_size = sizeof(uint16_t); + + return bit_width * line_size + exceptions_count * (exception_value_size + exception_position_size); +} + +template +bool is_compressable(const T* input_array, const size_t count) { + T* sample_array = new T[count]; + state alpstate; + + bool is_possible = false; + for (int32_t attempts = 0; attempts < MAX_ATTEMPTS_TO_ENCODE; ++attempts) { + alp::encoder::init(input_array, count, sample_array, alpstate); + + if ((is_possible = alpstate.scheme == alp::Scheme::ALP)) { + break; + } + } + + delete[] sample_array; + return is_possible; +} + +template +state configure_alpstate(const T* input_array, const size_t n_values) { + T* sample_array = new T[n_values]; + state alpstate; + + bool successful_encoding = false; + for (int32_t attempts = 0; attempts < MAX_ATTEMPTS_TO_ENCODE; ++attempts) { + alp::encoder::init(input_array, n_values, sample_array, alpstate); + + if ((successful_encoding = alpstate.scheme == Scheme::ALP)) { + break; + } + } + if (!successful_encoding) { + throw alp::EncodingException(); + } + delete[] sample_array; + return alpstate; +} + +template +flsgpu::host::ALPColumn encode(const T* input_array, const size_t n_values, const bool print_compression_info) { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + + const size_t n_vecs = utils::get_n_vecs_from_size(n_values); + state alpstate = configure_alpstate(input_array, n_values); + + // Intermediate arrays + UINT_T* packed_array = new UINT_T[consts::VALUES_PER_VECTOR]; + T* exceptions = new T[consts::VALUES_PER_VECTOR]; + uint16_t* positions = new uint16_t[consts::VALUES_PER_VECTOR]; + + auto v_packed_array = std::vector(); + auto v_exceptions = std::vector(); + auto v_positions = std::vector(); + + // Final arrays + vbw_t* bit_widths = new vbw_t[n_vecs]; + size_t* vector_offsets = new size_t[n_vecs]; + UINT_T* bases = new UINT_T[n_vecs]; + uint8_t* factor_indices = new uint8_t[n_vecs]; + uint8_t* fraction_indices = new uint8_t[n_vecs]; + size_t* exceptions_offsets = new size_t[n_vecs]; + uint16_t* counts = new uint16_t[n_vecs]; + + size_t compressed_vector_sizes = 0; + INT_T* encoded_array = new INT_T[consts::VALUES_PER_VECTOR]; + size_t exceptions_offset = 0; + size_t vector_offset = 0; + + size_t bit_widths_sum = 0; + for (size_t vi {0}; vi < n_vecs; vi++) { + alp::encoder::encode(input_array, exceptions, positions, encoded_array, alpstate, nullptr); + alp::encoder::analyze_ffor(encoded_array, bit_widths[vi], reinterpret_cast(&bases[vi])); + + counts[vi] = alpstate.n_exceptions; + bit_widths_sum += bit_widths[vi]; + fastlanes::generated::ffor::fallback::scalar::ffor( + reinterpret_cast(encoded_array), packed_array, bit_widths[vi], &bases[vi]); + + input_array += consts::VALUES_PER_VECTOR; + size_t compressed_values_size = utils::get_compressed_vector_size(bit_widths[vi]); + + v_exceptions.insert(v_exceptions.end(), exceptions, exceptions + counts[vi]); + v_positions.insert(v_positions.end(), positions, positions + counts[vi]); + + v_packed_array.insert(v_packed_array.end(), packed_array, packed_array + compressed_values_size); + + factor_indices[vi] = alpstate.fac; + fraction_indices[vi] = alpstate.exp; + exceptions_offsets[vi] = exceptions_offset; + exceptions_offset += counts[vi]; + vector_offsets[vi] = vector_offset; + vector_offset += compressed_values_size; + + compressed_vector_sizes += + alp::get_bytes_vector_compressed_size_without_overhead(bit_widths[vi], counts[vi]); + } + delete[] encoded_array; + + size_t compressed_alp_bytes_size = + compressed_vector_sizes + n_vecs * alp::get_bytes_overhead_size_per_alp_vector(); + size_t compressed_alp_extended_bytes_size = + compressed_vector_sizes + n_vecs * alp::get_bytes_overhead_size_per_alp_extended_vector(); + + // Wrapping it up + delete[] packed_array; + delete[] exceptions; + delete[] positions; + + packed_array = new UINT_T[v_packed_array.size()]; + exceptions = new T[v_exceptions.size()]; + positions = new uint16_t[v_positions.size()]; + + std::memcpy(packed_array, v_packed_array.data(), v_packed_array.size() * sizeof(UINT_T)); + std::memcpy(exceptions, v_exceptions.data(), v_exceptions.size() * sizeof(T)); + std::memcpy(positions, v_positions.data(), v_positions.size() * sizeof(uint16_t)); + + if (print_compression_info) { + const size_t input_size = n_values * sizeof(T); + printf("ALP_COMPRESSION_PARAMETERS,%zu,%f,%f,%f,%f\n", + n_vecs, + static_cast(input_size) / static_cast(compressed_alp_bytes_size), + static_cast(input_size) / static_cast(compressed_alp_extended_bytes_size), + static_cast(bit_widths_sum) / static_cast(n_vecs), + static_cast(v_exceptions.size()) / static_cast(n_vecs)); + } + + return flsgpu::host::ALPColumn { + flsgpu::host::FFORColumn { + flsgpu::host::BPColumn { + n_values, + v_packed_array.size(), + packed_array, + bit_widths, + vector_offsets, + }, + bases, + }, + factor_indices, + fraction_indices, + v_exceptions.size(), + exceptions_offsets, + exceptions, + positions, + counts, + compressed_alp_bytes_size, + compressed_alp_extended_bytes_size, + }; +} + +template +T* decode(const flsgpu::host::ALPColumn column, T* output_array) { + const size_t n_vecs = utils::get_n_vecs_from_size(column.ffor.bp.n_values); + + T* c_output_array = output_array; + for (size_t vi {0}; vi < n_vecs; ++vi) { + generated::falp::fallback::scalar::falp(column.ffor.bp.packed_array + column.ffor.bp.vector_offsets[vi], + c_output_array, + column.ffor.bp.bit_widths[vi], + &column.ffor.bases[vi], + column.factor_indices[vi], + column.fraction_indices[vi]); + + alp::state alpstate; + alpstate.n_exceptions = column.counts[vi]; + + alp::decoder::patch_exceptions(c_output_array, + column.exceptions + column.exceptions_offsets[vi], + column.positions + column.exceptions_offsets[vi], + alpstate); + + c_output_array += consts::VALUES_PER_VECTOR; + } + + return output_array; +} + +template +T* decode(const flsgpu::host::ALPExtendedColumn column, T* output_array) { + constexpr unsigned N_LANES = utils::get_n_lanes(); + const size_t n_vecs = utils::get_n_vecs_from_size(column.ffor.bp.n_values); + + T* c_output_array = output_array; + for (size_t vi {0}; vi < n_vecs; ++vi) { + generated::falp::fallback::scalar::falp(column.ffor.bp.packed_array + column.ffor.bp.vector_offsets[vi], + c_output_array, + column.ffor.bp.bit_widths[vi], + &column.ffor.bases[vi], + column.factor_indices[vi], + column.fraction_indices[vi]); + + // Reconstruct total count + uint16_t count = 0; + for (size_t offset_count_i {0}; offset_count_i < N_LANES; ++offset_count_i) { + count += column.offsets_counts[vi * N_LANES + offset_count_i] >> 10; + } + + alp::state alpstate; + alpstate.n_exceptions = count; // fix me + + alp::decoder::patch_exceptions(c_output_array, + column.exceptions + column.exceptions_offsets[vi], + column.positions + column.exceptions_offsets[vi], + alpstate); + + c_output_array += consts::VALUES_PER_VECTOR; + } + + return output_array; +} + +template bool is_compressable(const float* input_array, const size_t n_values); +template bool is_compressable(const double* input_array, const size_t n_values); + +template flsgpu::host::ALPColumn +encode(const float* input_array, const size_t n_values, const bool print_compression_info); +template flsgpu::host::ALPColumn +encode(const double* input_array, const size_t n_values, const bool print_compression_info); + +template float* decode(const flsgpu::host::ALPColumn column, float* output_array); +template double* decode(const flsgpu::host::ALPColumn column, double* output_array); +template float* decode(const flsgpu::host::ALPExtendedColumn column, float* output_array); +template double* decode(const flsgpu::host::ALPExtendedColumn column, double* output_array); + +} // namespace alp diff --git a/galp/src/engine/enums.cu b/galp/src/engine/enums.cu new file mode 100644 index 00000000..1d2f7fac --- /dev/null +++ b/galp/src/engine/enums.cu @@ -0,0 +1,163 @@ +#include "engine/enums.cuh" +#include + +namespace enums { +DataType string_to_data_type(const std::string& str) { + static const std::unordered_map mapping = { + {"u32", DataType::U32}, + {"u64", DataType::U64}, + {"f32", DataType::F32}, + {"f64", DataType::F64}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown kernel type: " + str); +} + +Kernel string_to_kernel(const std::string& str) { + static const std::unordered_map mapping = { + {"decompress", Kernel::Decompress}, + {"query", Kernel::Query}, + {"query-multi-column", Kernel::QueryMultiColumn}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown kernel type: " + str); +} + +Unpacker string_to_unpacker(const std::string& str) { + static const std::unordered_map mapping = { + {"dummy", Unpacker::Dummy}, + {"old-fls", Unpacker::OldFls}, + {"switch-case", Unpacker::SwitchCase}, + {"stateless", Unpacker::Stateless}, + {"stateless-branchless", Unpacker::StatelessBranchless}, + {"stateful-cache", Unpacker::StatefulCache}, + {"stateful-local-1", Unpacker::StatefulLocal1}, + {"stateful-local-2", Unpacker::StatefulLocal2}, + {"stateful-local-4", Unpacker::StatefulLocal4}, + {"stateful-shared-1", Unpacker::StatefulShared1}, + {"stateful-shared-2", Unpacker::StatefulShared2}, + {"stateful-shared-4", Unpacker::StatefulShared4}, + {"stateful-register-1", Unpacker::StatefulRegister1}, + {"stateful-register-2", Unpacker::StatefulRegister2}, + {"stateful-register-4", Unpacker::StatefulRegister4}, + {"stateful-register-branchless-1", Unpacker::StatefulRegisterBranchless1}, + {"stateful-register-branchless-2", Unpacker::StatefulRegisterBranchless2}, + {"stateful-register-branchless-4", Unpacker::StatefulRegisterBranchless4}, + {"stateful-branchless", Unpacker::StatefulBranchless}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown unpacker type: " + str); +} + +Patcher string_to_patcher(const std::string& str) { + static const std::unordered_map mapping = { + {"none", Patcher::None}, + {"dummy", Patcher::Dummy}, + {"stateless", Patcher::Stateless}, + {"stateful", Patcher::Stateful}, + {"naive", Patcher::Naive}, + {"naive-branchless", Patcher::NaiveBranchless}, + {"prefetch-position", Patcher::PrefetchPosition}, + {"prefetch-all", Patcher::PrefetchAll}, + {"prefetch-all-branchless", Patcher::PrefetchAllBranchless}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown patcher type: " + str); +} +} // namespace enums + +namespace enums_nvcomp { +ComparisonType string_to_comparison_type(const std::string& str) { + static const std::unordered_map mapping = { + {"decompression", ComparisonType::DECOMPRESSION}, + {"decompression_query", ComparisonType::DECOMPRESSION_QUERY}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown comparison type: " + str); +} + +std::string comparison_type_to_string(const ComparisonType type) { + switch (type) { + case ComparisonType::DECOMPRESSION: + return "decompression"; + case ComparisonType::DECOMPRESSION_QUERY: + return "decompression_query"; + default: + throw std::invalid_argument("Could not parse comparison type"); + } +} + +CompressionType string_to_compression_type(const std::string& str) { + static const std::unordered_map mapping = { + {"Thrust", CompressionType::THRUST}, + {"ALP", CompressionType::ALP}, + {"GALP", CompressionType::GALP}, + {"Bitcomp", CompressionType::BITCOMP}, + {"BitcompSparse", CompressionType::BITCOMP_SPARSE}, + {"LZ4", CompressionType::LZ4}, + {"zstd", CompressionType::ZSTD}, + {"Deflate", CompressionType::DEFLATE}, + {"GDeflate", CompressionType::GDEFLATE}, + {"Snappy", CompressionType::SNAPPY}, + }; + + auto it = mapping.find(str); + if (it != mapping.end()) { + return it->second; + } + + throw std::invalid_argument("Unknown compression type: " + str); +} + +std::string compression_type_to_string(const CompressionType type) { + switch (type) { + case CompressionType::THRUST: + return "Thrust"; + case CompressionType::ALP: + return "ALP"; + case CompressionType::GALP: + return "GALP"; + case CompressionType::BITCOMP: + return "Bitcomp"; + case CompressionType::BITCOMP_SPARSE: + return "BitcompSparse"; + case CompressionType::LZ4: + return "LZ4"; + case CompressionType::ZSTD: + return "zstd"; + case CompressionType::DEFLATE: + return "Deflate"; + case CompressionType::GDEFLATE: + return "GDeflate"; + case CompressionType::SNAPPY: + return "Snappy"; + default: + throw std::invalid_argument("Could not parse decompresor"); + } +} +} // namespace enums_nvcomp diff --git a/galp/src/include/alp/alp-bindings.cuh b/galp/src/include/alp/alp-bindings.cuh new file mode 100644 index 00000000..af0e986a --- /dev/null +++ b/galp/src/include/alp/alp-bindings.cuh @@ -0,0 +1,37 @@ +#ifndef ALP_BINDINGS_CUH +#define ALP_BINDINGS_CUH + +#include "alp.hpp" +#include "flsgpu/flsgpu-api.cuh" +#include +#include +#include +#include + +namespace alp { + +class EncodingException : public std::exception { +public: + using std::exception::what; + const char* what() { + return "Could not encode data with desired encoding."; + } +}; + +// Test if data can be decoded in specified type +template +bool is_compressable(const T* input_array, const size_t n_elements); + +template +flsgpu::host::ALPColumn +encode(const T* input_array, const size_t n_elements, const bool print_compression_info = false); + +template +T* decode(const flsgpu::host::ALPColumn column, T* output_array); + +template +T* decode(const flsgpu::host::ALPExtendedColumn column, T* output_array); + +} // namespace alp + +#endif // ALP_BINDINGS_CUH diff --git a/galp/src/include/engine/data.cuh b/galp/src/include/engine/data.cuh new file mode 100644 index 00000000..84668b4a --- /dev/null +++ b/galp/src/include/engine/data.cuh @@ -0,0 +1,545 @@ +#ifndef DATA_CUH +#define DATA_CUH + +#include "alp.hpp" +#include "alp/alp-bindings.cuh" +#include "fls/unffor.hpp" +#include "fls_gen/pack/pack.hpp" +#include "fls_gen/unpack/unpack.hpp" +#include "flsgpu/flsgpu-api.cuh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace data { + +namespace primitives { + +template +std::function get_random_number_generator(const T min, const T max) { + std::random_device random_device; + std::default_random_engine random_engine(random_device()); + std::uniform_int_distribution uniform_dist(min, max); + + return std::bind(uniform_dist, random_engine); +} + +template +T* fill_array_with_constant(T* array, const size_t n_values, const T value) { + for (size_t i {0}; i < n_values; ++i) { + array[i] = value; + } + + return array; +} + +template +T* fill_array_with_random_bytes(T* array, const size_t n_values, const unsigned repeat = 1) { + // WARNING Does not check if n_values % REPEAT == 0 + using UINT_T = typename utils::same_width_uint::type; + UINT_T* out = reinterpret_cast(array); + auto generator = + get_random_number_generator(std::numeric_limits::min(), std::numeric_limits::max()); + + for (size_t i {0}; i < n_values; i += repeat) { + UINT_T value = generator(); + + for (size_t r {0}; r < repeat; ++r) { + out[i + r] = value; + } + } + + return array; +} + +template +T* fill_array_with_sequence( + T* array, const size_t n_values, const T start, const T step = 1, const unsigned repeat = 1) { + // WARNING Does not check if n_values % REPEAT == 0 + using UINT_T = typename utils::same_width_uint::type; + UINT_T* out = reinterpret_cast(array); + T value = start; + + for (size_t i {0}; i < n_values; i += repeat) { + for (size_t r {0}; r < repeat; ++r) { + out[i + r] = value; + } + + value += step; + } + + return array; +} + +template +T* fill_array_with_random_data(T* array, + const size_t n_values, + const unsigned repeat = 1, + const T min = std::numeric_limits::min(), + const T max = std::numeric_limits::max()) { + // WARNING Does not check if n_values % REPEAT == 0 + using UINT_T = typename utils::same_width_uint::type; + UINT_T* out = reinterpret_cast(array); + auto generator = get_random_number_generator(min, max); + + for (size_t i {0}; i < n_values; i += repeat) { + UINT_T value = generator(); + + for (size_t r {0}; r < repeat; ++r) { + out[i + r] = value; + } + } + + return array; +} + +template +T_sum sum_array(const T* array, const size_t n_values) { + T_sum sum = 0; + + for (size_t i {0}; i < n_values; ++i) { + sum += static_cast(array[i]); + } + + return sum; +} + +template +T_out* prefix_sum_array(const T_in* in, T_out* out, const size_t n_values) { + T_out sum = 0; + T_out old_sum = 0; + + for (size_t i {0}; i < n_values; ++i) { + old_sum = sum; + sum += in[i]; + out[i] = old_sum; + } + + return out; +} + +template +T* map(LambdaT lambda, T* array, const size_t n_values) { + for (size_t i {0}; i < n_values; ++i) { + array[i] = lambda(array[i]); + } + + return array; +} + +template +std::tuple make_column_magic(T* data, const size_t n_values) { +} + +template +T* generate_positions(T* positions, const T* counts, const size_t n_vecs) { + // INFO Not really a primitive ... + T* indices = new T[consts::VALUES_PER_VECTOR]; + for (size_t i {0}; i < consts::VALUES_PER_VECTOR; ++i) { + indices[i] = i; + } + + // Shuffle them and copy to the positions array + std::random_device random_device; + auto rng = std::default_random_engine(random_device()); + T* c_positions = positions; + for (size_t vi {0}; vi < n_vecs; ++vi) { + std::shuffle(indices, indices + consts::VALUES_PER_VECTOR, rng); + std::memcpy(c_positions, indices, sizeof(T) * counts[vi]); + std::sort(c_positions, c_positions + counts[vi]); + c_positions += counts[vi]; + } + delete[] indices; + + return positions; +} + +} // namespace primitives + +template +struct ValueRange { + const T min; + const T max; + + ValueRange() + : min(std::numeric_limits::min()) + , max(std::numeric_limits::max()) { + } + ValueRange(const T a_value) + : min(a_value) + , max(a_value) { + } + ValueRange(const T a_start, const T a_end) + : min(a_start) + , max(std::max(a_start, a_end)) { + } + + ValueRange operator=(const ValueRange& other) { + return ValueRange(min, max); + } +}; + +namespace arrays { + +template +std::pair read_file_as(const std::string path, const size_t input_count) { + // Open file + std::ifstream inputFile(path, std::ios::binary | std::ios::ate); + if (!inputFile) { + throw std::invalid_argument("Could not open the specified file."); + } + // Get file size + const std::streamsize file_size = inputFile.tellg(); + inputFile.seekg(0, std::ios::beg); + + // Check file size to contain right type of data + bool file_size_is_multiple_of_T_size = static_cast(file_size) % static_cast(sizeof(T)) != 0; + if (file_size_is_multiple_of_T_size) { + throw std::invalid_argument("File size is incorrect, it is not a multiple of the type's size."); + } + + const size_t values_in_file = static_cast(file_size) / sizeof(T); + size_t count = input_count == 0 ? values_in_file : input_count; + count = count - (count % consts::VALUES_PER_VECTOR); + auto column = new T[count]; + + // Read either the file size, or the total number of values needed, + // whichever is smaller + const std::streamsize read_size = std::min(file_size, static_cast((count * sizeof(T)))); + if (!inputFile.read(reinterpret_cast(column), read_size)) { + throw std::invalid_argument("Failed to read file into column"); + } + + inputFile.close(); + + // Copy paste the values in file until the column is filled + if (values_in_file < count) { + size_t n_filled_values = values_in_file; + size_t n_empty_values_column = count - n_filled_values; + while (n_empty_values_column != 0) { + size_t n_values_to_copy = std::min(n_empty_values_column, values_in_file); + std::memcpy(column + n_filled_values, column, n_values_to_copy * sizeof(T)); + n_filled_values += n_values_to_copy; + n_empty_values_column -= n_values_to_copy; + } + } + + return std::make_pair(column, count); +} + +template +T* generate_index_array(const size_t n_values, const vbw_t value_bit_width) { + T mask = utils::h_set_first_n_bits(value_bit_width); + T* array = new T[n_values]; + + for (size_t i {0}; i < n_values; i++) { + array[i] = i & mask; + } + + return array; +} + +template +T* generate_random_array(const size_t n_values, const vbw_t value_bit_width) { + T max_value = utils::h_set_first_n_bits(value_bit_width); + return primitives::fill_array_with_random_data(new T[n_values], n_values, 1, T {0}, max_value); +} + +} // namespace arrays + +namespace bindings { + +template +flsgpu::host::BPColumn compress(const T* array, const size_t n_values, const vbw_t value_bit_width) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + size_t compressed_vector_size = utils::get_compressed_vector_size(value_bit_width); + size_t n_packed_values = n_vecs * compressed_vector_size; + + T* packed_array = new T[n_packed_values]; + T* c_packed_array = packed_array; + + for (size_t vi {0}; vi < n_vecs; ++vi) { + generated::pack::fallback::scalar::pack(array, c_packed_array, value_bit_width); + array += consts::VALUES_PER_VECTOR; + c_packed_array += compressed_vector_size; + } + + return flsgpu::host::BPColumn { + n_values, + n_packed_values, + packed_array, + primitives::fill_array_with_constant(new vbw_t[n_vecs], n_vecs, value_bit_width), + primitives::fill_array_with_sequence(new size_t[n_vecs], n_vecs, 0, compressed_vector_size), + }; +} + +template +T* decompress(const flsgpu::host::BPColumn column) { + T* out_array = new T[column.get_n_values()]; + T* c_out_array = out_array; + T* c_packed_array = column.packed_array; + + for (size_t vi {0}; vi < column.get_n_vecs(); ++vi) { + c_packed_array = column.packed_array + column.vector_offsets[vi]; + generated::unpack::fallback::scalar::unpack(c_packed_array, c_out_array, column.bit_widths[vi]); + c_out_array += consts::VALUES_PER_VECTOR; + } + + return out_array; +} + +template +T* decompress(const flsgpu::host::FFORColumn column) { + T* out_array = new T[column.get_n_values()]; + T* c_out_array = out_array; + T* c_packed_array = column.bp.packed_array + column.bp.vector_offsets[0]; + + for (size_t vi {0}; vi < column.get_n_vecs(); ++vi) { + c_packed_array = column.bp.packed_array + column.bp.vector_offsets[vi]; + fastlanes::generated::unffor::fallback::scalar::unffor( + c_packed_array, c_out_array, column.bp.bit_widths[vi], &column.bases[vi]); + c_out_array += consts::VALUES_PER_VECTOR; + } + + return out_array; +} + +template +T* decompress(const flsgpu::host::ALPColumn column) { + return alp::decode(column, new T[column.get_n_values()]); +} + +template +T* decompress(const flsgpu::host::ALPExtendedColumn column) { + return alp::decode(column, new T[column.get_n_values()]); +} + +} // namespace bindings + +namespace columns { + +template +flsgpu::host::BPColumn generate_index_bp_column(const size_t n_values, const vbw_t value_bit_width) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + size_t n_packed_values = n_vecs * utils::get_compressed_vector_size(value_bit_width); + T* array = arrays::generate_index_array(n_values, value_bit_width); + + auto column = bindings::compress(array, n_values, value_bit_width); + delete[] array; + return column; +} + +template +flsgpu::host::FFORColumn generate_index_ffor_column(const size_t n_values, const vbw_t value_bit_width) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + return flsgpu::host::FFORColumn { + generate_index_bp_column(n_values, value_bit_width), + primitives::fill_array_with_random_data(new T[n_vecs], n_vecs, 1, 0, 64), + }; +} + +template +flsgpu::host::BPColumn +generate_random_bp_column(const size_t n_values, const ValueRange value_bit_width, const int32_t repeat = 1) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + vbw_t* bit_widths = primitives::fill_array_with_random_data( + new vbw_t[n_vecs], n_vecs, repeat, value_bit_width.min, value_bit_width.max); + size_t n_packed_values = primitives::sum_array(bit_widths, n_vecs) * utils::get_n_lanes(); + + return flsgpu::host::BPColumn { + n_values, + n_packed_values, + primitives::fill_array_with_random_bytes(new T[n_packed_values], n_packed_values), + bit_widths, + primitives::map([](const size_t value) { return value * utils::get_n_lanes(); }, + primitives::prefix_sum_array(bit_widths, new size_t[n_vecs], n_vecs), + n_vecs), + }; +} + +template +flsgpu::host::FFORColumn generate_random_ffor_column(const size_t n_values, + const ValueRange value_bit_width, + const ValueRange bases, + const int32_t repeat = 1) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + return flsgpu::host::FFORColumn { + generate_random_bp_column(n_values, value_bit_width, repeat), + primitives::fill_array_with_random_data(new T[n_vecs], n_vecs, 1, bases.min, bases.max), + }; +} + +template +std::tuple> +generate_binary_bp_column(const size_t n_values, const ValueRange value_bit_width, const int32_t repeat = 1) { + T* data = primitives::fill_array_with_constant(new T[n_values], n_values, 0); + + auto generate_presence = primitives::get_random_number_generator(0, 100); + bool contains_magic_value = false; // generate_presence() < 50; + if (contains_magic_value) { + auto index_generator = primitives::get_random_number_generator(0, n_values - 1); + data[index_generator()] = consts::as::MAGIC_NUMBER; + } + + auto column = bindings::compress(data, n_values, value_bit_width.max); + delete data; + + return std::make_tuple(contains_magic_value, column); +} + +template +std::tuple> +generate_binary_ffor_column(const size_t n_values, const ValueRange value_bit_width, const int32_t repeat = 1) { + size_t n_vecs = utils::get_n_vecs_from_size(n_values); + auto [contains_magic_value, bp_column] = generate_binary_bp_column(n_values, value_bit_width, repeat); + T base = 0; + + if (value_bit_width.max == 0) { + base = T {contains_magic_value}; + } + + return std::make_tuple(contains_magic_value, + flsgpu::host::FFORColumn { + bp_column, + primitives::fill_array_with_constant(new T[n_vecs], n_vecs, base), + }); +} + +template +flsgpu::host::ALPColumn generate_alp_column(const size_t n_values, + const ValueRange bit_width_range, + const ValueRange exceptions_per_vec, + const unsigned repeat = 1) { + static_assert(std::is_floating_point::value, "T should be a floating point type."); + using UINT_T = typename utils::same_width_uint::type; + + const size_t n_vecs = utils::get_n_vecs_from_size(n_values); + auto column = flsgpu::host::ALPColumn(); + column.ffor = generate_random_ffor_column(n_values, bit_width_range, ValueRange(2, 20), repeat); + + // Note we halve the frac and fact because otherwise you + // are more likely to have integer overflow in the decoding for some + // combinations of compression parameters + column.factor_indices = primitives::fill_array_with_random_data( + new uint8_t[n_vecs], n_vecs, 1, 0, static_cast(consts::as::FACT_ARR_COUNT / 2)); + column.fraction_indices = primitives::fill_array_with_random_data( + new uint8_t[n_vecs], n_vecs, 1, 0, static_cast(consts::as::FRAC_ARR_COUNT / 2)); + + column.counts = primitives::fill_array_with_random_data( + new uint16_t[n_vecs], n_vecs, 1, exceptions_per_vec.min, exceptions_per_vec.max); + + column.n_exceptions = primitives::sum_array(column.counts, n_vecs); + column.exceptions_offsets = primitives::prefix_sum_array(column.counts, new size_t[n_vecs], n_vecs); + column.exceptions = primitives::fill_array_with_random_bytes(new T[column.n_exceptions], column.n_exceptions); + column.positions = + primitives::generate_positions(new uint16_t[column.n_exceptions], column.counts, n_vecs); + + // Not supported (yet) + column.compressed_size_bytes_alp = 0; + column.compressed_size_bytes_alp_extended = 0; + + return column; +} + +template +flsgpu::host::ALPColumn modify_alp_exception_count(flsgpu::host::ALPColumn column, + const ValueRange exceptions_per_vec) { + const size_t n_values = column.ffor.bp.n_values; + const size_t n_vecs = utils::get_n_vecs_from_size(n_values); + + delete[] column.counts; + delete[] column.exceptions_offsets; + delete[] column.exceptions; + delete[] column.positions; + + // Copied from generate_alp_column + column.counts = primitives::fill_array_with_random_data( + new uint16_t[n_vecs], n_vecs, 1, exceptions_per_vec.min, exceptions_per_vec.max); + column.n_exceptions = primitives::sum_array(column.counts, n_vecs); + column.exceptions_offsets = primitives::prefix_sum_array(column.counts, new size_t[n_vecs], n_vecs); + column.exceptions = primitives::fill_array_with_random_bytes(new T[column.n_exceptions], column.n_exceptions); + column.positions = + primitives::generate_positions(new uint16_t[column.n_exceptions], column.counts, n_vecs); + + return column; +} + +template +flsgpu::host::ALPColumn modify_alp_value_bit_width(flsgpu::host::ALPColumn column, + const ValueRange bit_width_range, + const unsigned repeat = 1) { + using UINT_T = typename utils::same_width_uint::type; + flsgpu::host::free_column(column.ffor); + column.ffor = generate_random_ffor_column( + column.ffor.bp.n_values, bit_width_range, repeat, ValueRange(2, 20)); + return column; +} + +template +void shuffle_bit_widths(flsgpu::host::BPColumn column) { + std::random_device random_device; + auto rng = std::default_random_engine(random_device()); + std::shuffle(column.bit_widths, column.bit_widths + column.get_n_vecs(), rng); + + column.vector_offsets = primitives::map( + [](const vbw_t value) { return value * utils::get_n_lanes(); }, + primitives::prefix_sum_array(column.bit_widths, column.vector_offsets, column.get_n_vecs()), + column.get_n_vecs()); +} + +template +void shuffle_bit_widths(flsgpu::host::FFORColumn column) { + shuffle_bit_widths(column.bp); +} + +template +void shuffle_bit_widths(flsgpu::host::ALPColumn column) { + shuffle_bit_widths(column.ffor.bp); +} + +template +void shuffle_bit_widths(flsgpu::host::ALPExtendedColumn column) { + shuffle_bit_widths(column.ffor.bp); +} + +template +std::tuple get_value_to_query(const ColumnT column) { + // Returns: + // The chance is 50% that the column contains the value. + // To find a suitable value: + // 50%) pick random value from column + // 50%) use constant, check whether it exists in the column + T* decompressed = bindings::decompress(column); + T value = consts::as::MAGIC_NUMBER; + bool column_does_not_contain_value = true; + + auto generate_presence = primitives::get_random_number_generator(0, 100); + bool ensure_column_contains_value = generate_presence() < 50; + if (ensure_column_contains_value) { + auto generate_index = primitives::get_random_number_generator(0, column.get_n_values() - 1); + value = decompressed[generate_index()]; + column_does_not_contain_value = false; + } else { + for (size_t i {0}; i < column.get_n_values(); ++i) { + column_does_not_contain_value &= value != decompressed[i]; + } + } + + delete decompressed; + return std::make_tuple(!column_does_not_contain_value, value); +} + +} // namespace columns + +} // namespace data + +#endif // DATA_CUH \ No newline at end of file diff --git a/galp/src/include/engine/device-utils.cuh b/galp/src/include/engine/device-utils.cuh new file mode 100644 index 00000000..18e59d1c --- /dev/null +++ b/galp/src/include/engine/device-utils.cuh @@ -0,0 +1,140 @@ +#include "flsgpu/flsgpu-api.cuh" +#include +#include + +#ifndef GPU_DEVICE_UTILS_CUH +#define GPU_DEVICE_UTILS_CUH + +template +struct SingleVectorPerWarpThreadblockMapping { + static constexpr unsigned N_WARPS_PER_BLOCK = std::max(utils::get_n_lanes() / consts::THREADS_PER_WARP, 2); + static constexpr unsigned N_THREADS_PER_BLOCK = N_WARPS_PER_BLOCK * consts::THREADS_PER_WARP; + static constexpr unsigned N_CONCURRENT_VECTORS_PER_BLOCK = + N_THREADS_PER_BLOCK / std::max(utils::get_n_lanes(), consts::THREADS_PER_WARP); + + const unsigned n_blocks; + + SingleVectorPerWarpThreadblockMapping(const size_t unpack_n_vecs, const size_t n_vecs) + : n_blocks(n_vecs / (unpack_n_vecs * N_CONCURRENT_VECTORS_PER_BLOCK)) { + } +}; +template +struct FillWarpThreadblockMapping { + static constexpr unsigned N_WARPS_PER_BLOCK = std::max(utils::get_n_lanes() / consts::THREADS_PER_WARP, 2); + static constexpr unsigned N_THREADS_PER_BLOCK = N_WARPS_PER_BLOCK * consts::THREADS_PER_WARP; + static constexpr unsigned N_CONCURRENT_VECTORS_PER_BLOCK = N_THREADS_PER_BLOCK / utils::get_n_lanes(); + + const unsigned n_blocks; + + FillWarpThreadblockMapping(const size_t unpack_n_vecs, const size_t n_vecs) + : n_blocks(n_vecs / (unpack_n_vecs * N_CONCURRENT_VECTORS_PER_BLOCK)) { + } +}; + +#ifdef SingleVectorMapping +template +using ThreadblockMapping = SingleVectorPerWarpThreadblockMapping; +#else +template +using ThreadblockMapping = FillWarpThreadblockMapping; +#endif + +struct VectorToWarpMappingBase { + virtual __device__ __forceinline__ lane_t get_lane() const; + virtual __device__ __forceinline__ vi_t get_vector_index() const; +}; + +template +struct SingleVectorPerWarpMapping : VectorToWarpMappingBase { + static constexpr uint32_t N_LANES = utils::get_n_lanes(); + static constexpr uint32_t N_VALUES_IN_LANE = utils::get_values_per_lane(); + + __device__ __forceinline__ lane_t get_lane() const override { + return threadIdx.x % N_LANES; + } + + __device__ __forceinline__ vi_t get_vector_index() const override { + // Concurrent vectors per block: how many vectors can be processed + // by the block simultaneously, assuming that each thread is 1 lane + + const int32_t concurrent_vectors_per_block = + blockDim.x / std::max(N_LANES, uint32_t {consts::THREADS_PER_WARP}); + const int32_t vectors_per_block = concurrent_vectors_per_block * UNPACK_N_VECTORS; + + const int32_t concurrent_vector_index = threadIdx.x / N_LANES; + const int32_t block_index = blockIdx.x; + + return vectors_per_block * block_index + concurrent_vector_index * UNPACK_N_VECTORS; + } +}; + +template +struct FillWarpMapping : VectorToWarpMappingBase { + static constexpr uint32_t N_LANES = utils::get_n_lanes(); + static constexpr uint32_t N_VALUES_IN_LANE = utils::get_values_per_lane(); + + __device__ __forceinline__ lane_t get_lane() const override { + return threadIdx.x % N_LANES; + } + + __device__ __forceinline__ vi_t get_vector_index() const override { + // Concurrent vectors per block: how many vectors can be processed + // by the block simultaneously, assuming that each thread is 1 lane + + const int32_t concurrent_vectors_per_block = blockDim.x / N_LANES; + const int32_t vectors_per_block = concurrent_vectors_per_block * UNPACK_N_VECTORS; + + const int32_t concurrent_vector_index = threadIdx.x / N_LANES; + const int32_t block_index = blockIdx.x; + + return vectors_per_block * block_index + concurrent_vector_index * UNPACK_N_VECTORS; + } +}; + +#ifdef SingleVectorMapping +template +using VectorToWarpMapping = OneVectorPerWarpMapping; +#else +template +using VectorToWarpMapping = FillWarpMapping; +#endif + +template +__device__ __forceinline__ void write_registers_to_global(const lane_t lane, + const si_t index_offset, + const T* __restrict registers, + T* __restrict out) { + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + for (int w {0}; w < UNPACK_N_VALUES; ++w) { + out[lane + (index_offset + w) * N_LANES + v * consts::VALUES_PER_VECTOR] = + registers[w + v * UNPACK_N_VALUES]; + } + } +} + +template +struct MagicChecker { + const T magic_value; + bool no_magic_found = true; + + __device__ __forceinline__ MagicChecker(const T magic_value) + : magic_value(magic_value) { + } + + __device__ __forceinline__ void check(const T* __restrict registers) { +#pragma unroll + for (int i = 0; i < N_VALUES; ++i) { + no_magic_found &= registers[i] != magic_value; + } + } + + __device__ __forceinline__ void write_result(bool* __restrict out) { + // This is a branch, as we do not want to write 0s, only emit a write + // if we found a magic value + if (!no_magic_found) { + *out = true; + } + } +}; + +#endif // GPU_DEVICE_UTILS_CUH diff --git a/galp/src/include/engine/enums.cuh b/galp/src/include/engine/enums.cuh new file mode 100644 index 00000000..f573a62a --- /dev/null +++ b/galp/src/include/engine/enums.cuh @@ -0,0 +1,94 @@ +#ifndef ENUMS_CUH +#define ENUMS_CUH + +#include +#include + +namespace enums { +enum class DataType { + U32, + U64, + F32, + F64, +}; + +enum class Kernel { + Decompress, + Query, + QueryMultiColumn, +}; + +enum class Unpacker { + Dummy, + OldFls, + SwitchCase, + Stateless, + StatelessBranchless, + StatefulCache, + StatefulLocal1, + StatefulLocal2, + StatefulLocal4, + StatefulShared1, + StatefulShared2, + StatefulShared4, + StatefulRegister1, + StatefulRegister2, + StatefulRegister4, + StatefulRegisterBranchless1, + StatefulRegisterBranchless2, + StatefulRegisterBranchless4, + StatefulBranchless, +}; + +enum class Patcher { + None, + Dummy, + Stateless, + Stateful, + Naive, + NaiveBranchless, + PrefetchPosition, + PrefetchAll, + PrefetchAllBranchless, +}; + +enum class Print { + PrintNothing, + PrintDebug, + PrintDebugExit0, +}; + +DataType string_to_data_type(const std::string& str); +Kernel string_to_kernel(const std::string& str); +Unpacker string_to_unpacker(const std::string& str); +Patcher string_to_patcher(const std::string& str); + +} // namespace enums + +namespace enums_nvcomp { +enum ComparisonType { + DECOMPRESSION, + DECOMPRESSION_QUERY, +}; + +ComparisonType string_to_comparison_type(const std::string& str); +std::string comparison_type_to_string(const ComparisonType type); + +enum CompressionType { + THRUST, + ALP, + GALP, + BITCOMP, + BITCOMP_SPARSE, + LZ4, + ZSTD, + DEFLATE, + GDEFLATE, + SNAPPY, +}; + +CompressionType string_to_compression_type(const std::string& str); +std::string compression_type_to_string(const CompressionType type); +} // namespace enums_nvcomp + +#endif // ENUMS_CUH diff --git a/galp/src/include/engine/kernels.cuh b/galp/src/include/engine/kernels.cuh new file mode 100644 index 00000000..884ff0a0 --- /dev/null +++ b/galp/src/include/engine/kernels.cuh @@ -0,0 +1,152 @@ +#include "engine/device-utils.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include + +#ifndef FLS_GLOBAL_CUH +#define FLS_GLOBAL_CUH + +namespace kernels { + +namespace device { + +template +__global__ void decompress_column(const ColumnT column, T* out) { + constexpr uint32_t N_VALUES = UNPACK_N_VALUES * UNPACK_N_VECTORS; + const auto mapping = VectorToWarpMapping(); + const lane_t lane = mapping.get_lane(); + const int32_t vector_index = mapping.get_vector_index(); + + T registers[N_VALUES]; + out += vector_index * consts::VALUES_PER_VECTOR; + + auto iterator = DecompressorT(column, vector_index, lane); + + for (si_t i = 0; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + iterator.unpack_next_into(registers); + + write_registers_to_global(lane, i, registers, out); + } +} + +template +__global__ void query_column(const ColumnT column, bool* out, const T magic_value) { + constexpr uint32_t N_VALUES = UNPACK_N_VALUES * UNPACK_N_VECTORS; + const auto mapping = VectorToWarpMapping(); + const lane_t lane = mapping.get_lane(); + const int32_t vector_index = mapping.get_vector_index(); + + T registers[N_VALUES]; + auto checker = MagicChecker(magic_value); + + DecompressorT unpacker = DecompressorT(column, vector_index, lane); + + for (si_t i = 0; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + unpacker.unpack_next_into(registers); + checker.check(registers); + } + + checker.write_result(out); +} + +template +__global__ void compute_column(const ColumnT column, bool* __restrict out, const T runtime_zero) { + constexpr T RANDOM_VALUE = 3; + constexpr uint32_t N_VALUES = UNPACK_N_VALUES * UNPACK_N_VECTORS; + const auto mapping = VectorToWarpMapping(); + const lane_t lane = mapping.get_lane(); + const vi_t vector_index = mapping.get_vector_index(); + + T registers[N_VALUES]; + auto checker = MagicChecker(1); + DecompressorT decompressor = DecompressorT(column, vector_index, lane); + + for (si_t i = 0; i < mapping.N_VALUES_IN_LANE; i += UNPACK_N_VALUES) { + decompressor.unpack_next_into(registers); + +#pragma unroll + for (int32_t j {0}; j < N_VALUES; ++j) { +#pragma unroll + for (int32_t k {0}; k < N_REPETITIONS; ++k) { + registers[j] *= RANDOM_VALUE; + registers[j] <<= RANDOM_VALUE; + registers[j] += RANDOM_VALUE; + registers[j] >>= RANDOM_VALUE; + registers[j] &= runtime_zero; + } + } + + checker.check(registers); + } + + checker.write_result(out); +} +} // namespace device + +namespace host { + +template +__host__ T* decompress_column(const ColumnT column, const uint32_t n_samples) { + size_t n_vecs = utils::get_n_vecs_from_size(column.n_values); + const ThreadblockMapping mapping(UNPACK_N_VECTORS, n_vecs); + GPUArray device_out(column.n_values); + + for (uint32_t i {0}; i < n_samples; ++i) { + device::decompress_column + <<>>(column, device_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + T* out = new T[column.n_values]; + device_out.copy_to_host(out); + return out; +} + +template +__host__ bool query_column(const ColumnT column, const T magic_value, const uint32_t n_samples) { + size_t n_vecs = utils::get_n_vecs_from_size(column.n_values); + const ThreadblockMapping mapping(UNPACK_N_VECTORS, n_vecs); + bool result = false; + GPUArray device_out(1, &result); + + for (uint32_t i {0}; i < n_samples; ++i) { + device::query_column + <<>>(column, device_out.get(), magic_value); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + device_out.copy_to_host(&result); + return result; +} + +template +__host__ bool compute_column(const ColumnT column, const uint32_t n_samples) { + size_t n_vecs = utils::get_n_vecs_from_size(column.n_values); + const ThreadblockMapping mapping(UNPACK_N_VECTORS, n_vecs); + GPUArray device_out(1); + bool result; + + for (uint32_t i {0}; i < n_samples; ++i) { + device::compute_column + <<>>(column, device_out.get(), 0); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + device_out.copy_to_host(&result); + return result; +} + +} // namespace host + +} // namespace kernels + +#endif // FLS_GLOBAL_CUH diff --git a/galp/src/include/engine/multi-column-host-kernels.cuh b/galp/src/include/engine/multi-column-host-kernels.cuh new file mode 100644 index 00000000..89f94d73 --- /dev/null +++ b/galp/src/include/engine/multi-column-host-kernels.cuh @@ -0,0 +1,141 @@ +#ifndef MULTI_COLUMN_HOST_KERNELS_CUH +#define MULTI_COLUMN_HOST_KERNELS_CUH + +#include "data.cuh" +#include "device-utils.cuh" +#include "generated-bindings/multi-column-device-kernels.cuh" + +namespace kernels { namespace host { + +template +__host__ bool query_multi_column(const ColumnT column, const T magic_value, const uint32_t n_samples) { + using DeviceColumnT = typename ColumnT::DeviceColumnT; + constexpr int32_t MAX_N_COLS = 10; + DeviceColumnT device_columns[MAX_N_COLS]; + + for (int32_t c {0}; c < MAX_N_COLS; ++c) { + // INFO: Bitwidths are shuffled to ensure that the vectors that + // are unpacked in the same loop do not have an identical bitwidth, + // as this might benefit branched unpackers + data::columns::shuffle_bit_widths(column); + device_columns[c] = column.copy_to_device(); + } + + bool result = false; + GPUArray d_out(1, &result); + const ThreadblockMapping mapping(UNPACK_N_VECS, column.get_n_vecs()); + + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>( + device_columns[0], device_columns[1], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>( + device_columns[0], device_columns[1], device_columns[2], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>( + device_columns[0], device_columns[1], device_columns[2], device_columns[3], magic_value, d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + device_columns[7], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + device_columns[7], + device_columns[8], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + for (uint32_t repeat {0}; repeat < n_samples; ++repeat) { + multi_column::query_multi_column + <<>>(device_columns[0], + device_columns[1], + device_columns[2], + device_columns[3], + device_columns[4], + device_columns[5], + device_columns[6], + device_columns[7], + device_columns[8], + device_columns[9], + magic_value, + d_out.get()); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + } + + for (int32_t c {0}; c < MAX_N_COLS; ++c) { + flsgpu::host::free_column(device_columns[c]); + } + + d_out.copy_to_host(&result); + return result; +} +}} // namespace kernels::host +#endif // MULTI_COLUMN_HOST_KERNELS_CUH diff --git a/galp/src/include/engine/verification.cuh b/galp/src/include/engine/verification.cuh new file mode 100644 index 00000000..6f23cfde --- /dev/null +++ b/galp/src/include/engine/verification.cuh @@ -0,0 +1,118 @@ +#ifndef VERIFICATION_H +#define VERIFICATION_H + +#include "flsgpu/flsgpu-api.cuh" +#include +#include +#include + +namespace verification { + +constexpr size_t LOG_N_MISTAKES = 5; + +template +struct Difference { + size_t index; + T original; + T other; + + template ::value, bool> = true> + void log() { + fprintf(stderr, + "[%lu] correct: %lu, found: %lu\n", + static_cast(index), + static_cast(original), + static_cast(other)); + } + + template ::value, bool> = true> + void log() { + using UINT_T = typename utils::same_width_uint::type; + + UINT_T* original_c = reinterpret_cast(&original); + UINT_T* other_c = reinterpret_cast(&other); + + if (sizeof(U) == 8) { + fprintf(stderr, + "[%lu] correct: %f (%016lX), found: %f (%016lX)\n", + index, + static_cast(original), + static_cast(*original_c), + static_cast(other), + static_cast(*other_c)); + } else { + fprintf(stderr, + "[%lu] correct: %f (%08X), found: %f (%08X)\n", + index, + static_cast(original), + static_cast(*original_c), + static_cast(other), + static_cast(*other_c)); + } + } +}; + +template +struct ExecutionResult { + bool success; + std::vector> differences; +}; + +template +bool byte_compare(const T a, const T b) { + using UINT_T = typename utils::same_width_uint::type; + const UINT_T* a_c = reinterpret_cast(&a); + const UINT_T* b_c = reinterpret_cast(&b); + + return (*a_c) == (*b_c); +} + +template +ExecutionResult compare_data(const T* a, const T* b, const size_t size) { + auto differences = std::vector>(); + + for (size_t i {0}; i < size; ++i) { + if (!byte_compare(a[i], b[i])) { + differences.push_back(Difference {i, a[i], b[i]}); + + if (differences.size() > LOG_N_MISTAKES) { + break; + } + } + } + + return ExecutionResult {differences.size() == 0, differences}; +} + +template +int32_t process_results(std::vector> results, bool print_debug) { + + int32_t runs_failed = 0; + for (size_t i {0}; i < results.size(); i++) { + if (!results[i].success) { + ++runs_failed; + + if (print_debug) { + fprintf(stderr, "\n Run %lu failed.\n", i); + + for (auto difference : results[i].differences) { + difference.template log(); + } + } + } + } + + if (print_debug) { + if (runs_failed == 0) { + fprintf(stderr, "Compression successful.\n"); + } else { + fprintf(stderr, "\n[%d/%ld] Runs failed.\n", runs_failed, results.size()); + } + } + + return runs_failed; +} + +} // namespace verification + +#endif // VERIFICATION_H diff --git a/galp/src/include/flsgpu/alp.cuh b/galp/src/include/flsgpu/alp.cuh new file mode 100644 index 00000000..d8b0a45b --- /dev/null +++ b/galp/src/include/flsgpu/alp.cuh @@ -0,0 +1,461 @@ +#ifndef ALP_CUH +#define ALP_CUH + +#include "fls.cuh" +#include "structs.cuh" +#include +#include +#include + +namespace flsgpu { namespace device { + +template +struct ALPFunctor : FunctorBase { +private: + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + + UINT_T bases[UNPACK_N_VECTORS]; + INT_T factor[UNPACK_N_VECTORS]; + T frac10[UNPACK_N_VECTORS]; + +public: + __device__ __forceinline__ ALPFunctor(const UINT_T* a_bases, + const INT_T* factors, + const uint8_t* factor_indices, + const T* fractions, + const uint8_t* fraction_indices) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + bases[v] = a_bases[v]; + factor[v] = factors[factor_indices[v]]; + frac10[v] = fractions[fraction_indices[v]]; + } + } + + __device__ __forceinline__ T operator()(const UINT_T value, const vi_t vector_index) override { + return static_cast(static_cast((value + bases[vector_index]) * factor[vector_index])) * + frac10[vector_index]; + } +}; + +template +struct ALPExceptionPatcherBase { +public: + virtual void __device__ __forceinline__ patch_if_needed(T* out); +}; + +template +struct DummyALPExceptionPatcher : flsgpu::device::ALPExceptionPatcherBase { + +public: + void __device__ __forceinline__ patch_if_needed(T* out) override { + } + + __device__ __forceinline__ + DummyALPExceptionPatcher(const flsgpu::device::ALPColumn column, const vi_t vector_index, const lane_t lane) { + } +}; + +template +struct StatelessALPExceptionPatcher : ALPExceptionPatcherBase { + using INT_T = typename utils::same_width_int::type; + + si_t start_index = 0; + uint16_t exceptions_count[UNPACK_N_VECTORS]; + uint16_t* vec_exceptions_positions[UNPACK_N_VECTORS]; + T* vec_exceptions[UNPACK_N_VECTORS]; + const lane_t lane; + +public: + void __device__ __forceinline__ patch_if_needed(T* out) override { + constexpr auto N_LANES = utils::get_n_lanes(); + + const int first_pos = start_index * N_LANES + lane; + const int last_pos = first_pos + N_LANES * (UNPACK_N_VALUES - 1); + + start_index += UNPACK_N_VALUES; + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + for (int i {0}; i < exceptions_count[v]; i++) { + auto position = vec_exceptions_positions[v][i]; + auto exception = vec_exceptions[v][i]; + if (position >= first_pos) { + if (position <= last_pos && position % N_LANES == lane) { + out[(position - first_pos) / N_LANES + v * UNPACK_N_VALUES] = exception; + } + if (position + 1 > last_pos) { + break; + } + } + } + } + } + + __device__ __forceinline__ + StatelessALPExceptionPatcher(const ALPColumn column, const vi_t first_vector_index, const lane_t lane) + : lane(lane) { + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + auto vec_index = first_vector_index + v; + exceptions_count[v] = column.counts[vec_index]; + vec_exceptions_positions[v] = column.positions + column.exceptions_offsets[vec_index]; + vec_exceptions[v] = column.exceptions + column.exceptions_offsets[vec_index]; + } + } +}; + +template +struct StatefulALPExceptionPatcher : ALPExceptionPatcherBase { + using INT_T = typename utils::same_width_int::type; + + si_t start_index = 0; + uint16_t exceptions_count[UNPACK_N_VECTORS]; + uint16_t* vec_exceptions_positions[UNPACK_N_VECTORS]; + T* vec_exceptions[UNPACK_N_VECTORS]; + const lane_t lane; + int32_t exception_index[UNPACK_N_VECTORS] = {0}; + +public: + void __device__ __forceinline__ patch_if_needed(T* out) override { + constexpr auto N_LANES = utils::get_n_lanes(); + + const int first_pos = start_index * N_LANES + lane; + const int last_pos = first_pos + N_LANES * (UNPACK_N_VALUES - 1); + start_index += UNPACK_N_VALUES; + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + for (; exception_index[v] < exceptions_count[v]; exception_index[v]++) { + auto position = vec_exceptions_positions[v][exception_index[v]]; + auto exception = vec_exceptions[v][exception_index[v]]; + if (position >= first_pos) { + if (position <= last_pos && position % N_LANES == lane) { + out[(position - first_pos) / N_LANES + v * UNPACK_N_VALUES] = exception; + } + if (position + 1 > last_pos) { + break; + } + } + } + } + } + + __device__ __forceinline__ + StatefulALPExceptionPatcher(const ALPColumn column, const vi_t first_vector_index, const lane_t lane) + : lane(lane) { + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + auto vec_index = first_vector_index + v; + exceptions_count[v] = column.counts[vec_index]; + vec_exceptions_positions[v] = column.positions + column.exceptions_offsets[vec_index]; + vec_exceptions[v] = column.exceptions + column.exceptions_offsets[vec_index]; + } + } +}; + +template +struct NaiveALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + uint16_t current_position; + +public: + __device__ __forceinline__ + NaiveALPExceptionPatcher(const ALPExtendedColumn column, const vi_t vector_index, const lane_t lane) + : current_position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t current_vector_index = vector_index + v; + + const auto offset_count = column.offsets_counts[current_vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto offset = (offset_count & 0x3FF); + positions[v] = column.positions + column.exceptions_offsets[current_vector_index] + offset; + exceptions[v] = column.exceptions + column.exceptions_offsets[current_vector_index] + offset; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + if (count[v] > 0 && current_position == *(positions[v])) { + out[v * UNPACK_N_VALUES + w] = *(exceptions[v]); + ++(positions[v]); + ++(exceptions[v]); + --(count[v]); + } + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +constexpr ToT __device__ __forceinline__ reinterpret_as(FromT value) { + ToT* ptr = reinterpret_cast(&value); + return *ptr; +} + +template +constexpr void __device__ __forceinline__ +overwrite_if_true(T* __restrict buffer, const T* __restrict new_value, const bool condition) { + using UINT_T = typename utils::same_width_uint::type; + *buffer = reinterpret_as((reinterpret_as(*buffer) * (!condition)) | + (reinterpret_as(*new_value) * condition)); +} + +template +struct NaiveBranchlessALPExceptionPatcher : ALPExceptionPatcherBase { +private: + using UINT_T = typename utils::same_width_uint::type; + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + uint16_t next_position; + uint16_t current_position; + +public: + __device__ __forceinline__ + NaiveBranchlessALPExceptionPatcher(const ALPExtendedColumn column, const vi_t vector_index, const lane_t lane) + : current_position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t current_vector_index = vector_index + v; + + const auto offset_count = column.offsets_counts[current_vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[current_vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + // It is possible to easily prefetch *positions[v] here + bool comp = (count[v] > 0) && (current_position == (*positions[v])); + overwrite_if_true(&out[v * UNPACK_N_VALUES + w], exceptions[v], comp); + positions[v] += comp; + exceptions[v] += comp; + count[v] -= comp; + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +struct PrefetchPositionALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + uint16_t next_position[UNPACK_N_VECTORS]; + uint16_t position; + +public: + __device__ __forceinline__ PrefetchPositionALPExceptionPatcher(const ALPExtendedColumn column, + const vi_t first_vector_index, + const lane_t lane) + : position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t vector_index = first_vector_index + v; + const auto offset_count = column.offsets_counts[vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + + next_position[v] = *positions[v]; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + if (count[v] > 0 && position == next_position[v]) { + out[v * UNPACK_N_VALUES + w] = *exceptions[v]; + ++positions[v]; + ++exceptions[v]; + --count[v]; + next_position[v] = *positions[v]; + } + } + position += utils::get_n_lanes(); + } + } +}; + +template +struct PrefetchAllALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + + uint16_t index[UNPACK_N_VECTORS] = {0}; + uint16_t next_position[UNPACK_N_VECTORS]; + T next_exception[UNPACK_N_VECTORS]; + uint16_t current_position; + +public: + void __device__ __forceinline__ read_next_exception(vi_t v) { + if (index[v] < count[v]) { + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + ++positions[v]; + ++exceptions[v]; + ++index[v]; + } else { + next_position[v] = consts::VALUES_PER_VECTOR; + } + } + + __device__ __forceinline__ + PrefetchAllALPExceptionPatcher(const ALPExtendedColumn column, const vi_t first_vector_index, const lane_t lane) + : current_position(lane) { + // Parse the data from the column +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const auto vector_index = first_vector_index + v; + const auto offset_count = column.offsets_counts[vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + + // This might be avoided to avoid a branch + read_next_exception(v); + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + if (current_position == next_position[v]) { + out[v * UNPACK_N_VALUES + w] = next_exception[v]; + read_next_exception(v); + } + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +struct PrefetchAllBranchlessALPExceptionPatcher : ALPExceptionPatcherBase { +private: + uint16_t count[UNPACK_N_VECTORS]; + uint16_t* positions[UNPACK_N_VECTORS]; + T* exceptions[UNPACK_N_VECTORS]; + + uint16_t index[UNPACK_N_VECTORS] = {0}; + uint16_t next_position[UNPACK_N_VECTORS]; + T next_exception[UNPACK_N_VECTORS]; + uint16_t current_position; + +public: + void __device__ __forceinline__ read_next_exception() { + } + + __device__ __forceinline__ PrefetchAllBranchlessALPExceptionPatcher(const ALPExtendedColumn column, + const vi_t first_vector_index, + const lane_t lane) + : current_position(lane) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + const vi_t vector_index = first_vector_index + v; + const auto offset_count = column.offsets_counts[vector_index * utils::get_n_lanes() + lane]; + count[v] = offset_count >> 10; + + const auto exceptions_offset = column.exceptions_offsets[vector_index]; + const auto lane_offset = (offset_count & 0x3FF); + positions[v] = column.positions + exceptions_offset + lane_offset; + exceptions[v] = column.exceptions + exceptions_offset + lane_offset; + + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + + bool comparison = count[v] > 0; + next_position[v] += (!comparison) * consts::VALUES_PER_VECTOR; + } + } + + void __device__ __forceinline__ patch_if_needed(T* out) override { + // NOTES: It is probably possible to remove the next_position variable + // as well as the next_exception, if you use prefetching. This would make + // it easier to do multiple vectors. + +#pragma unroll + for (int w {0}; w < UNPACK_N_VALUES; ++w) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + bool comparison = current_position == next_position[v]; + overwrite_if_true(out + v * UNPACK_N_VALUES + w, &next_exception[v], comparison); + + positions[v] += comparison; + exceptions[v] += comparison; + next_position[v] = *positions[v]; + next_exception[v] = *exceptions[v]; + index[v] += comparison; + + comparison = index[v] < count[v]; + next_position[v] += (!comparison) * consts::VALUES_PER_VECTOR; + } + current_position += utils::get_n_lanes(); + } + } +}; + +template +struct ALPDecompressor : DecompressorBase { + UnpackerT unpacker; + PatcherT patcher; + + __device__ __forceinline__ ALPDecompressor(const ColumnT column, const vi_t vector_index, const lane_t lane) + : unpacker(column.ffor.bp.packed_array + column.ffor.bp.vector_offsets[vector_index], + lane, + column.ffor.bp.bit_widths[vector_index], + ALPFunctor(column.ffor.bases + vector_index, + column.factors, + column.factor_indices + vector_index, + column.fractions, + column.fraction_indices + vector_index)) + , patcher(PatcherT(column, vector_index, lane)) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + patcher.patch_if_needed(out); + } +}; + +}} // namespace flsgpu::device +#endif // ALP_CUH diff --git a/galp/src/include/flsgpu/consts.cuh b/galp/src/include/flsgpu/consts.cuh new file mode 100644 index 00000000..ca0f70db --- /dev/null +++ b/galp/src/include/flsgpu/consts.cuh @@ -0,0 +1,33 @@ +#ifndef CONSTS_H +#define CONSTS_H + +#include + +namespace consts { + +constexpr int32_t REGISTER_WIDTH = 1024; +constexpr int32_t VALUES_PER_VECTOR = 1024; +constexpr int32_t THREADS_PER_WARP = 32; +constexpr unsigned MAX_UNPACK_N_VECS = 4; + +template +struct as { + static inline constexpr T MAGIC_NUMBER = 1; +}; + +template <> +struct as { + static inline constexpr float MAGIC_NUMBER = 0.3214f; + static inline constexpr int32_t FACT_ARR_COUNT = 10; + static inline constexpr int32_t FRAC_ARR_COUNT = 11; +}; + +template <> +struct as { + static inline constexpr double MAGIC_NUMBER = 0.3214; + static inline constexpr int32_t FACT_ARR_COUNT = 19; + static inline constexpr int32_t FRAC_ARR_COUNT = 21; +}; +} // namespace consts + +#endif // CONSTS_H diff --git a/galp/src/include/flsgpu/device-types.cuh b/galp/src/include/flsgpu/device-types.cuh new file mode 100644 index 00000000..ae8d203e --- /dev/null +++ b/galp/src/include/flsgpu/device-types.cuh @@ -0,0 +1,11 @@ +#ifndef GPU_DEVICE_TYPES_CUH +#define GPU_DEVICE_TYPES_CUH + +#include + +using vbw_t = uint8_t; +using lane_t = uint16_t; +using si_t = uint32_t; // si = start index (of value within vector) +using vi_t = uint32_t; // vi = Vector Index + +#endif // GPU_DEVICE_TYPES_CUH diff --git a/galp/src/include/flsgpu/fls-switch-case.cuh b/galp/src/include/flsgpu/fls-switch-case.cuh new file mode 100644 index 00000000..efe13320 --- /dev/null +++ b/galp/src/include/flsgpu/fls-switch-case.cuh @@ -0,0 +1,891 @@ +#ifndef FLS_SWITCH_CASE_CUH +#define FLS_SWITCH_CASE_CUH + +#include "fls.cuh" + +namespace flsgpu { namespace device { + +template +__device__ void +switch_leaf(const typename utils::same_width_uint::type* __restrict in, T* __restrict out, processor_T processor) { + using UINT_T = typename utils::same_width_uint::type; + constexpr uint8_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + constexpr uint32_t N_LANES = utils::get_n_lanes(); + constexpr uint16_t PRECEDING_BITS = (START_INDEX * VALUE_BIT_WIDTH); + constexpr uint16_t BUFFER_OFFSET = PRECEDING_BITS % LANE_BIT_WIDTH; + constexpr uint16_t N_INPUT_LINE = PRECEDING_BITS / LANE_BIT_WIDTH; + + LoaderT loader(in + N_INPUT_LINE * N_LANES, utils::get_compressed_vector_size(VALUE_BIT_WIDTH)); + + constexpr T VALUE_MASK = utils::h_set_first_n_bits(VALUE_BIT_WIDTH); + + Masker masker(BUFFER_OFFSET, VALUE_BIT_WIDTH); + + UINT_T values[UNPACK_N_VECTORS]; + + // INFO Yes these lines contain if statements without constexpr + // INFO No this is not a problem, as you can check with a profiler + // whether the compiler is able to emit the right SASS, and the SASS + // is correct, with no run time branches, only 5-8 instructions per case + +#pragma unroll + for (int i = 0; i < UNPACK_N_VALUES; ++i) { + if (masker.is_buffer_empty()) { + loader.next_line(); + masker.next_line(); + } + + UINT_T buffers[UNPACK_N_VECTORS]; + loader.load_next_into(buffers); + masker.mask_and_increment(values, buffers); + + if (masker.continues_on_next_line()) { + loader.next_line(); + masker.next_line(); + loader.load_next_into(buffers); + masker.mask_and_insert_remaining_value(values, buffers); + } + +#pragma unroll + for (int v = 0; v < UNPACK_N_VECTORS; ++v) { + *(out + i + v * UNPACK_N_VALUES) = processor(values[v], v); + } + } +} + +template +__device__ void switch_start_index(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + processor_T processor, + const si_t start_index) { + constexpr int N_VALUES_IN_LANE = utils::get_values_per_lane(); + + switch (start_index) { + case 0: + if constexpr (0 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 1: + if constexpr (1 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 2: + if constexpr (2 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 3: + if constexpr (3 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 4: + if constexpr (4 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 5: + if constexpr (5 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 6: + if constexpr (6 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 7: + if constexpr (7 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 8: + if constexpr (8 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 9: + if constexpr (9 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 10: + if constexpr (10 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 11: + if constexpr (11 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 12: + if constexpr (12 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 13: + if constexpr (13 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 14: + if constexpr (14 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 15: + if constexpr (15 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 16: + if constexpr (16 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 17: + if constexpr (17 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 18: + if constexpr (18 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 19: + if constexpr (19 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 20: + if constexpr (20 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 21: + if constexpr (21 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 22: + if constexpr (22 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 23: + if constexpr (23 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 24: + if constexpr (24 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 25: + if constexpr (25 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 26: + if constexpr (26 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 27: + if constexpr (27 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 28: + if constexpr (28 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 29: + if constexpr (29 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 30: + if constexpr (30 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 31: + if constexpr (31 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 32: + if constexpr (32 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 33: + if constexpr (33 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 34: + if constexpr (34 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 35: + if constexpr (35 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 36: + if constexpr (36 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 37: + if constexpr (37 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 38: + if constexpr (38 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 39: + if constexpr (39 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 40: + if constexpr (40 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 41: + if constexpr (41 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 42: + if constexpr (42 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 43: + if constexpr (43 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 44: + if constexpr (44 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 45: + if constexpr (45 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 46: + if constexpr (46 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 47: + if constexpr (47 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 48: + if constexpr (48 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 49: + if constexpr (49 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 50: + if constexpr (50 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 51: + if constexpr (51 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 52: + if constexpr (52 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 53: + if constexpr (53 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 54: + if constexpr (54 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 55: + if constexpr (55 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 56: + if constexpr (56 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 57: + if constexpr (57 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 58: + if constexpr (58 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 59: + if constexpr (59 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 60: + if constexpr (60 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 61: + if constexpr (61 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 62: + if constexpr (62 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + case 63: + if constexpr (63 < N_VALUES_IN_LANE) { + switch_leaf( + in, out, processor); + } + break; + } +} + +template +__device__ void switch_value_bit_width(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + processor_T processor, + const vbw_t value_bit_width, + const si_t start_index) { + constexpr int N_BITS = utils::sizeof_in_bits(); + + switch (value_bit_width) { + case 0: + if constexpr (0 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 1: + if constexpr (1 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 2: + if constexpr (2 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 3: + if constexpr (3 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 4: + if constexpr (4 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 5: + if constexpr (5 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 6: + if constexpr (6 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 7: + if constexpr (7 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 8: + if constexpr (8 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 9: + if constexpr (9 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 10: + if constexpr (10 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 11: + if constexpr (11 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 12: + if constexpr (12 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 13: + if constexpr (13 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 14: + if constexpr (14 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 15: + if constexpr (15 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 16: + if constexpr (16 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 17: + if constexpr (17 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 18: + if constexpr (18 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 19: + if constexpr (19 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 20: + if constexpr (20 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 21: + if constexpr (21 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 22: + if constexpr (22 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 23: + if constexpr (23 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 24: + if constexpr (24 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 25: + if constexpr (25 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 26: + if constexpr (26 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 27: + if constexpr (27 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 28: + if constexpr (28 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 29: + if constexpr (29 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 30: + if constexpr (30 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 31: + if constexpr (31 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 32: + if constexpr (32 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 33: + if constexpr (33 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 34: + if constexpr (34 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 35: + if constexpr (35 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 36: + if constexpr (36 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 37: + if constexpr (37 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 38: + if constexpr (38 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 39: + if constexpr (39 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 40: + if constexpr (40 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 41: + if constexpr (41 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 42: + if constexpr (42 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 43: + if constexpr (43 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 44: + if constexpr (44 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 45: + if constexpr (45 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 46: + if constexpr (46 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 47: + if constexpr (47 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 48: + if constexpr (48 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 49: + if constexpr (49 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 50: + if constexpr (50 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 51: + if constexpr (51 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 52: + if constexpr (52 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 53: + if constexpr (53 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 54: + if constexpr (54 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 55: + if constexpr (55 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 56: + if constexpr (56 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 57: + if constexpr (57 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 58: + if constexpr (58 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 59: + if constexpr (59 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 60: + if constexpr (60 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 61: + if constexpr (61 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 62: + if constexpr (62 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 63: + if constexpr (63 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + case 64: + if constexpr (64 <= N_BITS) { + switch_start_index( + in, out, processor, start_index); + } + break; + } +} + +template +struct BitUnpackerSwitchCase : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* __restrict in; + const vbw_t value_bit_width; + OutputProcessor processor; + si_t start_index = 0; + + __device__ __forceinline__ BitUnpackerSwitchCase(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(in + lane) + , value_bit_width(value_bit_width) + , processor(processor) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + switch_value_bit_width>( + in, out, processor, value_bit_width, start_index); + start_index += UNPACK_N_VALUES; + } +}; + +}} // namespace flsgpu::device + +#endif // FLS_SWITCH_CASE_CUH diff --git a/galp/src/include/flsgpu/fls.cuh b/galp/src/include/flsgpu/fls.cuh new file mode 100644 index 00000000..0e74883b --- /dev/null +++ b/galp/src/include/flsgpu/fls.cuh @@ -0,0 +1,648 @@ +#ifndef FLS_CUH +#define FLS_CUH + +#include "device-types.cuh" +#include "old-fls.cuh" +#include "structs.cuh" +#include "utils.cuh" +#include +#include +#include +#include + +namespace flsgpu { namespace device { + +template +struct BPFunctor : FunctorBase { + using UINT_T = typename utils::same_width_uint::type; + __device__ __forceinline__ BPFunctor() {}; + __device__ __forceinline__ T operator()(const UINT_T value, [[maybe_unused]] const vi_t vector_index) override { + return value; + } +}; + +template +struct FFORFunctor : FunctorBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T bases[UNPACK_N_VECTORS]; + __device__ __forceinline__ FFORFunctor(const UINT_T* a_bases) { +#pragma unroll + for (int32_t v {0}; v < UNPACK_N_VECTORS; ++v) { + bases[v] = a_bases[v]; + } + }; + + __device__ __forceinline__ UINT_T operator()(const UINT_T value, const vi_t vector_index) override { + return value + bases[vector_index]; + } +}; + +template +struct BitUnpackerBase { + /* Constructor, but cannot be enforced + BitUnpackerBase( + const UINT_T *__restrict in, const lane_t lane, + const vbw_t value_bit_width, OutputProcessor processor) + */ + virtual __device__ __forceinline__ void unpack_next_into(T* __restrict out); +}; + +template +struct BitUnpackerDummy : flsgpu::device::BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* in; + OutputProcessor processor; + + __device__ __forceinline__ BitUnpackerDummy(const UINT_T* __restrict a_in, + const lane_t lane, + [[maybe_unused]] const vbw_t value_bit_width, + OutputProcessor processor) + : in(a_in + lane) + , processor(processor) {}; + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + constexpr int32_t N_LANES = utils::get_n_lanes(); + +#pragma unroll + for (int v = 0; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int j = 0; j < UNPACK_N_VALUES; ++j) { + out[v * UNPACK_N_VALUES + j] = processor(in[v * consts::VALUES_PER_VECTOR + j * N_LANES], v); + } + } + + in += UNPACK_N_VALUES * N_LANES; + } +}; + +template +struct BitUnpackerOldFls : flsgpu::device::BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* in; + const vbw_t value_bit_width; + OutputProcessor processor; + + __device__ __forceinline__ BitUnpackerOldFls(const UINT_T* __restrict a_in, + const lane_t lane, + const vbw_t a_value_bit_width, + OutputProcessor processor) + : in(a_in + lane) + , value_bit_width(a_value_bit_width) + , processor(processor) { + static_assert(UNPACK_N_VECTORS == 1, "Old FLS can only unpack 1 at a time"); + static_assert(UNPACK_N_VALUES == utils::get_values_per_lane(), "Old FLS can only unpack entire lanes"); + }; + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + UINT_T* u_out = reinterpret_cast(out); + oldfls::adjusted::unpack(in, u_out, value_bit_width); + + for (int32_t i {0}; i < UNPACK_N_VALUES; ++i) { + out[i] = processor(u_out[i], 0); + } + } +}; + +template +struct LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + + virtual __device__ __forceinline__ void load_next_into(UINT_T* out); + virtual __device__ __forceinline__ void next_line(); +}; + +template +struct CacheLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* in; + int32_t vector_offset; + + __device__ __forceinline__ CacheLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) {}; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = *(in + v * vector_offset); + } + } + + __device__ __forceinline__ void next_line() override { + in += utils::get_n_lanes(); + } +}; + +template +struct LocalMemoryLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T buffers[UNPACK_N_VECTORS * BUFFER_SIZE]; + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + + __device__ __forceinline__ LocalMemoryLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + buffer_index]; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { + ++buffer_index; + } + } +}; + +template +struct SharedMemoryLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + // No syncthreads are needed as threads only read and write their own section + // Shared memory is allocated per block, so this also depends on block config + // 4 divide by 32 bits/lanes, multiply by number of warps per block + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + UINT_T* buffers; + + __device__ __forceinline__ SharedMemoryLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + constexpr uint32_t N_LANES = utils::get_n_lanes(); + __shared__ UINT_T shared_ptr[N_LANES * BUFFER_SIZE * UNPACK_N_VECTORS * (sizeof(T) / 4 * 2)]; + buffers = shared_ptr + threadIdx.x * BUFFER_SIZE * UNPACK_N_VECTORS; + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + buffer_index]; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { + ++buffer_index; + } + } +}; + +template +struct RegisterLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T buffers[UNPACK_N_VECTORS * BUFFER_SIZE]; + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + + __device__ __forceinline__ RegisterLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + static_assert(BUFFER_SIZE <= 4, "Switch in RegisterLoader is not long enough for this buffer size."); + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { + + switch (buffer_index) { + case 0: { + if (0 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 0]; + } + } + } break; + case 1: { + if (1 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 1]; + } + } + } break; + case 2: { + if (2 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 2]; + } + } + } break; + case 3: { + if (3 < BUFFER_SIZE) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE + 3]; + } + } + } break; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { + ++buffer_index; + } + } +}; + +template +struct RegisterBranchlessLoader : LoaderBase { + using UINT_T = typename utils::same_width_uint::type; + UINT_T buffers[UNPACK_N_VECTORS * BUFFER_SIZE]; + const UINT_T* in; + int32_t vector_offset; + int32_t buffer_index = BUFFER_SIZE; + + __device__ __forceinline__ RegisterBranchlessLoader(const UINT_T* in, const int32_t vector_offset) + : in(in) + , vector_offset(vector_offset) { + next_line(); + }; + + __device__ __forceinline__ void load_next_into(UINT_T* out) override { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + out[v] = buffers[v * BUFFER_SIZE]; + } + } + + __device__ __forceinline__ void next_line() override { + if (buffer_index >= BUFFER_SIZE - 1) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {0}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b] = *(in + v * vector_offset + b * utils::get_n_lanes()); + } + } + in += BUFFER_SIZE * utils::get_n_lanes(); + buffer_index = 0; + } else { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { +#pragma unroll + for (int b {1}; b < BUFFER_SIZE; ++b) { + buffers[v * BUFFER_SIZE + b - 1] = buffers[v * BUFFER_SIZE + b]; + } + } + ++buffer_index; + } + } +}; + +template +struct Masker { + using UINT_T = typename utils::same_width_uint::type; + const vbw_t value_bit_width; + const T value_mask; + uint16_t buffer_offset = 0; + + __device__ __forceinline__ Masker(const vbw_t value_bit_width) + : value_bit_width(value_bit_width) + , value_mask(utils::set_first_n_bits(value_bit_width)) {}; + + __device__ __forceinline__ Masker(const uint16_t buffer_offset, const vbw_t value_bit_width) + : buffer_offset(buffer_offset) + , value_bit_width(value_bit_width) + , value_mask(utils::set_first_n_bits(value_bit_width)) {}; + + __device__ __forceinline__ void mask_and_increment(T* values, const T* buffers) { +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + values[v] = (buffers[v] & (value_mask << buffer_offset)) >> buffer_offset; + } + buffer_offset += value_bit_width; + } + + __device__ __forceinline__ void next_line() { + buffer_offset -= utils::get_lane_bitwidth(); + } + __device__ __forceinline__ bool is_buffer_empty() const { + return buffer_offset == utils::get_lane_bitwidth(); + } + + __device__ __forceinline__ bool continues_on_next_line() const { + return buffer_offset > utils::get_lane_bitwidth(); + } + + __device__ __forceinline__ void mask_and_insert_remaining_value(T* values, const T* buffers) const { + T buffer_offset_mask = (T {1} << static_cast(buffer_offset)) - T {1}; + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + values[v] |= (buffers[v] & buffer_offset_mask) << (value_bit_width - buffer_offset); + } + } +}; + +template +__device__ void unpack_vector_stateless(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + const lane_t lane, + const vbw_t value_bit_width, + const si_t start_index, + processor_T processor, + int32_t vector_offset) { + using UINT_T = typename utils::same_width_uint::type; + constexpr uint8_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + constexpr uint32_t N_LANES = utils::get_n_lanes(); + uint16_t preceding_bits = (start_index * value_bit_width); + uint16_t buffer_offset = preceding_bits % LANE_BIT_WIDTH; + uint16_t n_input_line = preceding_bits / LANE_BIT_WIDTH; + + LoaderT loader(in + n_input_line * N_LANES + lane, vector_offset); + Masker masker(buffer_offset, value_bit_width); + + UINT_T values[UNPACK_N_VECTORS]; + +#pragma unroll + for (int i = 0; i < UNPACK_N_VALUES; ++i) { + if (masker.is_buffer_empty()) { + loader.next_line(); + masker.next_line(); + } + + UINT_T buffers[UNPACK_N_VECTORS]; + loader.load_next_into(buffers); + masker.mask_and_increment(values, buffers); + + if (masker.continues_on_next_line()) { + loader.next_line(); + masker.next_line(); + loader.load_next_into(buffers); + masker.mask_and_insert_remaining_value(values, buffers); + } + +#pragma unroll + for (int v = 0; v < UNPACK_N_VECTORS; ++v) { + *(out + i + v * UNPACK_N_VALUES) = processor(values[v], v); + } + } +} + +template +struct BitUnpackerStateless : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* __restrict in; + const lane_t lane; + const vbw_t value_bit_width; + OutputProcessor processor; + int32_t vector_offset; + + si_t start_index = 0; + + __device__ __forceinline__ BitUnpackerStateless(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(in) + , lane(lane) + , value_bit_width(value_bit_width) + , processor(processor) + , vector_offset(utils::get_compressed_vector_size(value_bit_width)) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + unpack_vector_stateless>( + in, out, lane, value_bit_width, start_index, processor, vector_offset); + start_index += UNPACK_N_VALUES; + } +}; + +template +__device__ void unpack_vector_stateless_branchless(const typename utils::same_width_uint::type* __restrict in, + T* __restrict out, + const lane_t lane, + const vbw_t value_bit_width, + const si_t start_index, + processor_T processor, + const int32_t vector_offset) { + using UINT_T = typename utils::same_width_uint::type; + constexpr int32_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + constexpr int32_t N_LANES = utils::get_n_lanes(); + constexpr int32_t BIT_COUNT = utils::sizeof_in_bits(); + + int32_t preceding_bits_first = (start_index * value_bit_width); + int32_t n_input_line = preceding_bits_first / LANE_BIT_WIDTH; + int32_t offset_first = preceding_bits_first % LANE_BIT_WIDTH; + int32_t offset_second = BIT_COUNT - offset_first; + UINT_T value_mask = utils::set_first_n_bits(value_bit_width); + + UINT_T values[UNPACK_N_VECTORS] = {0}; + + in += n_input_line * N_LANES + lane; +#pragma unroll + for (int32_t v {0}; v < UNPACK_N_VECTORS; v++) { + const auto v_in = in + v * vector_offset; + values[v] |= (v_in[0] & (value_mask << offset_first)) >> offset_first; + values[v] |= (v_in[N_LANES] & (value_mask >> offset_second)) << offset_second; + out[v] = processor(values[v], v); + } +} + +template +struct BitUnpackerStatelessBranchless : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + + const UINT_T* __restrict in; + const lane_t lane; + const vbw_t value_bit_width; + OutputProcessor processor; + const int32_t vector_offset; + + si_t start_index = 0; + + __device__ __forceinline__ BitUnpackerStatelessBranchless(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(in) + , lane(lane) + , value_bit_width(value_bit_width) + , processor(processor) + , vector_offset(utils::get_compressed_vector_size(value_bit_width)) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { +#pragma unroll + for (int32_t i {0}; i < UNPACK_N_VALUES; i++) { + unpack_vector_stateless_branchless( + in, out + i, lane, value_bit_width, start_index + i, processor, vector_offset); + } + start_index += UNPACK_N_VALUES; + } +}; + +template +struct BitUnpackerStateful : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + LoaderT loader; + Masker masker; + OutputProcessor processor; + + __device__ __forceinline__ BitUnpackerStateful(const UINT_T* __restrict in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : loader(in + lane, utils::get_compressed_vector_size(value_bit_width)) + , masker(value_bit_width) + , processor(processor) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + UINT_T values[UNPACK_N_VECTORS]; + +#pragma unroll + for (int i = 0; i < UNPACK_N_VALUES; ++i) { + if (masker.is_buffer_empty()) { + loader.next_line(); + masker.next_line(); + } + + UINT_T buffers[UNPACK_N_VECTORS]; + loader.load_next_into(buffers); + masker.mask_and_increment(values, buffers); + + if (masker.continues_on_next_line()) { + loader.next_line(); + masker.next_line(); + loader.load_next_into(buffers); + masker.mask_and_insert_remaining_value(values, buffers); + } + +#pragma unroll + for (int v {0}; v < UNPACK_N_VECTORS; ++v) { + *(out + i + v * UNPACK_N_VALUES) = processor(values[v], v); + } + } + } +}; + +template +struct BitUnpackerStatefulBranchless : BitUnpackerBase { + using UINT_T = typename utils::same_width_uint::type; + OutputProcessor processor; + + const UINT_T* in; + const int32_t vector_offset; + const vbw_t value_bit_width; + + int32_t offset_first = 0; + UINT_T value_mask; + + __device__ __forceinline__ BitUnpackerStatefulBranchless(const UINT_T* __restrict a_in, + const lane_t lane, + const vbw_t value_bit_width, + OutputProcessor processor) + : in(a_in + lane) + , value_bit_width(value_bit_width) + , value_mask(utils::set_first_n_bits(value_bit_width)) + , vector_offset(utils::get_compressed_vector_size(value_bit_width)) + , processor(processor) { + } + + __device__ __forceinline__ void unpack_next_into(T* __restrict out) override { + constexpr int32_t N_LANES = utils::get_n_lanes(); + constexpr int32_t BIT_COUNT = utils::sizeof_in_bits(); + constexpr int32_t LANE_BIT_WIDTH = utils::get_lane_bitwidth(); + +#pragma unroll + for (int32_t i {0}; i < UNPACK_N_VALUES; i++) { + const auto offset_second = BIT_COUNT - offset_first; + +#pragma unroll + for (int32_t v {0}; v < UNPACK_N_VECTORS; v++) { + const auto v_in = in + v * vector_offset; + out[UNPACK_N_VALUES * v + i] = + processor(((v_in[0] >> offset_first) & value_mask) | + ((v_in[N_LANES] & (value_mask >> offset_second)) << offset_second), + v); + } + + in += (offset_second <= value_bit_width) * N_LANES; + offset_first = (offset_first + value_bit_width) % LANE_BIT_WIDTH; + } + } +}; + +template +struct BPDecompressor : DecompressorBase { + UnpackerT unpacker; + __device__ __forceinline__ BPDecompressor(const BPColumn column, const vi_t vector_index, const lane_t lane) + : unpacker(column.packed_array + column.vector_offsets[vector_index], + lane, + column.bit_widths[vector_index], + BPFunctor()) { + } + + void __device__ unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + } +}; + +template +struct FFORDecompressor : DecompressorBase { + UnpackerT unpacker; + __device__ __forceinline__ FFORDecompressor(const FFORColumn column, const vi_t vector_index, const lane_t lane) + : unpacker(column.bp.packed_array + column.bp.vector_offsets[vector_index], + lane, + column.bp.bit_widths[vector_index], + FFORFunctor(column.bases + vector_index)) { + } + + void __device__ unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + } +}; + +}} // namespace flsgpu::device + +#endif // FLS_CUH diff --git a/galp/src/include/flsgpu/flsgpu-api.cuh b/galp/src/include/flsgpu/flsgpu-api.cuh new file mode 100644 index 00000000..3926ee6d --- /dev/null +++ b/galp/src/include/flsgpu/flsgpu-api.cuh @@ -0,0 +1,4 @@ +#include "alp.cuh" +#include "fls-switch-case.cuh" +#include "fls.cuh" +#include "structs.cuh" diff --git a/galp/src/include/flsgpu/host-utils.cuh b/galp/src/include/flsgpu/host-utils.cuh new file mode 100644 index 00000000..159e99d5 --- /dev/null +++ b/galp/src/include/flsgpu/host-utils.cuh @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifndef GPU_UTILS_H +#define GPU_UTILS_H + +#define CUDA_SAFE_CALL(call) \ + do { \ + cudaError_t err = call; \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in file '%s' in line %i : %s.", __FILE__, __LINE__, cudaGetErrorString(err)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define CUDA_SAFE_CALL_TRACED(call) \ + do { \ + fprintf(stderr, "Start CUDA_CALL ['%s': line %i\n", __FILE__, __LINE__); \ + cudaError_t err = call; \ + fprintf(stderr, "End CUDA_CALL ['%s': line %i\n", __FILE__, __LINE__); \ + if (cudaSuccess != err) { \ + fprintf(stderr, "Cuda error in file '%s' in line %i : %s.", __FILE__, __LINE__, cudaGetErrorString(err)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +template +void free_device_pointer(T*& device_ptr) { + if (device_ptr != nullptr) { + CUDA_SAFE_CALL(cudaFree(device_ptr)); + } + device_ptr = nullptr; +} + +template +class GPUArray { +private: + size_t allocation_size; + size_t memory_size; + T* device_ptr = nullptr; + + void allocate() { + CUDA_SAFE_CALL(cudaMalloc(reinterpret_cast(&device_ptr), allocation_size)); + } + +public: + GPUArray(const size_t count) { + memory_size = count * sizeof(T); + allocation_size = memory_size; + allocate(); + } + + GPUArray(const size_t count, const T* host_p) { + memory_size = count * sizeof(T); + allocation_size = memory_size; + allocate(); + CUDA_SAFE_CALL(cudaMemcpy(device_ptr, host_p, memory_size, cudaMemcpyHostToDevice)); + } + + GPUArray(const size_t count, const size_t buffer, const T* host_p) { + memory_size = count * sizeof(T); + allocation_size = memory_size + buffer * sizeof(T); + allocate(); + + CUDA_SAFE_CALL(cudaMemcpy(device_ptr, host_p, memory_size, cudaMemcpyHostToDevice)); + } + + // Copy constructor + GPUArray(const GPUArray&) = delete; + // Assignment operator deleted + GPUArray& operator=(const GPUArray&) = delete; + + // Move constructor + GPUArray(GPUArray&& other) noexcept + : device_ptr(other.device_ptr) { + other.device_ptr = nullptr; + } + + // Assignment operator + GPUArray& operator=(GPUArray&& other) noexcept { + if (this != &other) { + free_device_pointer(device_ptr); + device_ptr = other.device_ptr; + other.device_ptr = nullptr; + } + return *this; + } + + ~GPUArray() { + free_device_pointer(device_ptr); + } + + void copy_to_host(T* host_p) { + CUDA_SAFE_CALL(cudaMemcpy(host_p, device_ptr, memory_size, cudaMemcpyDeviceToHost)); + } + + T* get() { + return device_ptr; + } + + T* release() { + auto temp = device_ptr; + device_ptr = nullptr; + return temp; + } +}; + +#endif // GPU_UTILS_H diff --git a/galp/src/include/flsgpu/old-fls.cuh b/galp/src/include/flsgpu/old-fls.cuh new file mode 100644 index 00000000..4b0af3bb --- /dev/null +++ b/galp/src/include/flsgpu/old-fls.cuh @@ -0,0 +1,6936 @@ +#ifndef OLD_FLS_KERNELS_H +#define OLD_FLS_KERNELS_H + +#include +#include +#include + +namespace oldfls { + +namespace original { +__device__ __forceinline__ void unpack_0bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + // int i = threadIdx.x; // THREAD INDEX + + out[0] = base_0; + out[1] = base_0; + out[2] = base_0; + out[3] = base_0; + out[4] = base_0; + out[5] = base_0; + out[6] = base_0; + out[7] = base_0; + out[8] = base_0; + out[9] = base_0; + out[10] = base_0; + out[11] = base_0; + out[12] = base_0; + out[13] = base_0; + out[14] = base_0; + out[15] = base_0; + out[16] = base_0; + out[17] = base_0; + out[18] = base_0; + out[19] = base_0; + out[20] = base_0; + out[21] = base_0; + out[22] = base_0; + out[23] = base_0; + out[24] = base_0; + out[25] = base_0; + out[26] = base_0; + out[27] = base_0; + out[28] = base_0; + out[29] = base_0; + out[30] = base_0; + out[31] = base_0; +} +__device__ __forceinline__ void unpack_1bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 1) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_2bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_3bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 3) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_4bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_5bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 5) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_6bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_7bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 7) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_8bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_9bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 9) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_10bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_11bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 11) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_12bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_13bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 13) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_14bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_15bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 15) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_16bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[1] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[5] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[9] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[13] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[17] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[21] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[25] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[29] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_17bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 17) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; + out[18] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; + out[22] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; + out[26] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; + out[30] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_18bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[10] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[14] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[26] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[30] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_19bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 19) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; + out[6] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; + out[18] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; + out[26] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; + out[30] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_20bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[6] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[14] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[22] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[30] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_21bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 21) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; + out[10] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; + out[18] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; + out[22] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; + out[30] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_22bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[2] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[14] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[18] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[30] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_23bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 23) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; + out[2] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; + out[6] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; + out[26] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; + out[30] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_24bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[2] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[6] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[10] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[14] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[18] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[22] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[26] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[30] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_25bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 25) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; + out[2] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; + out[6] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; + out[12] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; + out[26] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; + out[30] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_26bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[2] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[4] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[14] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[18] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[20] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[30] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_27bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 27) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; + out[4] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; + out[10] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; + out[18] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; + out[22] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; + out[30] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_28bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[6] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[14] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[22] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[30] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_29bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 29) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; + out[6] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; + out[18] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; + out[20] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; + out[26] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; + out[30] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_30bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[10] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[12] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[14] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[26] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[28] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[30] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_31bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 31) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; + out[18] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; + out[22] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; + out[26] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; + out[28] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 960); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; + out[30] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_32bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = threadIdx.x; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + out[0] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + out[1] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + out[2] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + out[3] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + out[4] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + out[5] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + out[6] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + out[7] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + out[8] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + out[9] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + out[10] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + out[11] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + out[12] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + out[13] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + out[14] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + out[15] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 512); + out[16] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 544); + out[17] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + out[18] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 608); + out[19] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 640); + out[20] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + out[21] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 704); + out[22] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 736); + out[23] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 768); + out[24] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 800); + out[25] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 832); + out[26] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 864); + out[27] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 896); + out[28] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 928); + out[29] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 960); + out[30] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 992); + out[31] = register_0; +} +__device__ __forceinline__ void unpack(const uint32_t* __restrict a_in_p, uint32_t* __restrict a_out_p, uint8_t bw) { + switch (bw) { + case 0: + unpack_0bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 1: + unpack_1bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 2: + unpack_2bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 3: + unpack_3bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 4: + unpack_4bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 5: + unpack_5bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 6: + unpack_6bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 7: + unpack_7bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 8: + unpack_8bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 9: + unpack_9bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 10: + unpack_10bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 11: + unpack_11bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 12: + unpack_12bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 13: + unpack_13bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 14: + unpack_14bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 15: + unpack_15bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 16: + unpack_16bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 17: + unpack_17bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 18: + unpack_18bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 19: + unpack_19bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 20: + unpack_20bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 21: + unpack_21bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 22: + unpack_22bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 23: + unpack_23bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 24: + unpack_24bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 25: + unpack_25bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 26: + unpack_26bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 27: + unpack_27bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 28: + unpack_28bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 29: + unpack_29bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 30: + unpack_30bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 31: + unpack_31bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 32: + unpack_32bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + } +} + +} // namespace original + +namespace adjusted { + +__device__ __forceinline__ void unpack_0bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + // int i = 0; // THREAD INDEX + + out[0] = base_0; + out[1] = base_0; + out[2] = base_0; + out[3] = base_0; + out[4] = base_0; + out[5] = base_0; + out[6] = base_0; + out[7] = base_0; + out[8] = base_0; + out[9] = base_0; + out[10] = base_0; + out[11] = base_0; + out[12] = base_0; + out[13] = base_0; + out[14] = base_0; + out[15] = base_0; + out[16] = base_0; + out[17] = base_0; + out[18] = base_0; + out[19] = base_0; + out[20] = base_0; + out[21] = base_0; + out[22] = base_0; + out[23] = base_0; + out[24] = base_0; + out[25] = base_0; + out[26] = base_0; + out[27] = base_0; + out[28] = base_0; + out[29] = base_0; + out[30] = base_0; + out[31] = base_0; +} +__device__ __forceinline__ void unpack_1bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 1) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_2bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 2) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_3bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 3) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_4bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 4) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_5bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 5) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_6bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 6) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_7bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 7) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_8bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 8) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_9bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 9) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_10bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 10) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_11bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 11) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_12bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 12) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_13bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 13) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_14bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 14) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_15bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 15) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_16bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[1] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[5] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[9] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[13] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[17] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[21] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[25] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[29] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 16) - 1); + out[30] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_17bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 17) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; + out[1] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; + out[5] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; + out[9] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; + out[13] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; + out[18] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; + out[22] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; + out[26] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; + out[30] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_18bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[1] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[5] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[10] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[14] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 18) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; + out[17] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; + out[21] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; + out[26] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; + out[30] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_19bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 19) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; + out[1] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; + out[5] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; + out[6] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; + out[10] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; + out[13] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; + out[18] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; + out[21] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; + out[25] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; + out[26] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; + out[30] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_20bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[1] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[6] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[9] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[14] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[17] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[22] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 20) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; + out[25] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; + out[30] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_21bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 21) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; + out[1] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); + out[2] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; + out[6] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; + out[9] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; + out[10] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; + out[13] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; + out[18] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; + out[21] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; + out[22] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; + out[25] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); + out[29] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; + out[30] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_22bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[1] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[2] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[5] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[10] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[13] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[14] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 = (register_0) & ((1ULL << 22) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; + out[17] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; + out[18] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; + out[21] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; + out[26] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; + out[29] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; + out[30] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_23bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 23) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; + out[1] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; + out[2] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; + out[5] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; + out[6] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; + out[9] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; + out[13] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); + out[14] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); + out[17] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; + out[18] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; + out[22] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; + out[25] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; + out[26] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; + out[29] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; + out[30] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_24bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[1] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[2] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[3] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[5] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[6] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[9] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[10] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[11] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[13] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[14] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[17] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[18] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[19] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[20] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[21] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[22] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[25] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[26] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[27] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 24) - 1); + out[28] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; + out[29] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; + out[30] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_25bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 25) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; + out[1] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; + out[2] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; + out[3] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); + out[4] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; + out[5] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; + out[6] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); + out[9] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; + out[10] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; + out[11] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; + out[12] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); + out[13] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; + out[14] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; + out[17] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); + out[18] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; + out[19] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; + out[20] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; + out[21] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); + out[22] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; + out[25] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; + out[26] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); + out[27] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; + out[28] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; + out[29] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; + out[30] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_26bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[1] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[2] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[3] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[4] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[5] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[6] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[9] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[11] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[12] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[13] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[14] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 = (register_0) & ((1ULL << 26) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; + out[17] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; + out[18] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; + out[19] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; + out[20] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; + out[22] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; + out[25] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); + out[26] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; + out[27] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; + out[28] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; + out[29] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; + out[30] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_27bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 27) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; + out[1] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; + out[2] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; + out[3] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; + out[4] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; + out[5] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); + out[6] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; + out[9] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; + out[10] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; + out[11] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); + out[12] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; + out[13] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; + out[14] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; + out[17] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; + out[18] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); + out[19] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; + out[20] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; + out[21] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; + out[22] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); + out[25] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; + out[26] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; + out[27] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; + out[28] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; + out[29] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; + out[30] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_28bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[1] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[2] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[3] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[4] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[5] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[6] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[7] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[8] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[9] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[10] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[11] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[12] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[13] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[14] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[17] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[18] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[19] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[20] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[21] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[22] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[23] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 = (register_0) & ((1ULL << 28) - 1); + out[24] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; + out[25] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; + out[26] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; + out[27] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; + out[28] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; + out[29] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; + out[30] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_29bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 29) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; + out[1] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; + out[2] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; + out[3] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; + out[4] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; + out[5] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; + out[6] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; + out[7] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; + out[8] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; + out[9] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); + out[10] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; + out[11] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; + out[12] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; + out[13] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; + out[14] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; + out[17] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; + out[18] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; + out[19] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; + out[20] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); + out[21] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; + out[22] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; + out[23] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; + out[24] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; + out[25] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; + out[26] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; + out[27] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; + out[28] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; + out[29] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; + out[30] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_30bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[1] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[2] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[3] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[4] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[5] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[6] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[7] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[8] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[9] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[10] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[11] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[12] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[13] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[14] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[15] = tmp_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 = (register_0) & ((1ULL << 30) - 1); + out[16] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; + out[17] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; + out[18] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; + out[19] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; + out[20] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; + out[21] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; + out[22] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; + out[23] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; + out[24] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; + out[25] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; + out[26] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; + out[27] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; + out[28] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; + out[29] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; + out[30] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_31bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + tmp_0 = (register_0) & ((1ULL << 31) - 1); + out[0] = tmp_0; + tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 32); + tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; + out[1] = tmp_0; + tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 64); + tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; + out[2] = tmp_0; + tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 96); + tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; + out[3] = tmp_0; + tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 128); + tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; + out[4] = tmp_0; + tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 160); + tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; + out[5] = tmp_0; + tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 192); + tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; + out[6] = tmp_0; + tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 224); + tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; + out[7] = tmp_0; + tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 256); + tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; + out[8] = tmp_0; + tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 288); + tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; + out[9] = tmp_0; + tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 320); + tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; + out[10] = tmp_0; + tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 352); + tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; + out[11] = tmp_0; + tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 384); + tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; + out[12] = tmp_0; + tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 416); + tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; + out[13] = tmp_0; + tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 448); + tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; + out[14] = tmp_0; + tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 480); + tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; + out[15] = tmp_0; + tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 512); + tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; + out[16] = tmp_0; + tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 544); + tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; + out[17] = tmp_0; + tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 576); + tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; + out[18] = tmp_0; + tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 608); + tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; + out[19] = tmp_0; + tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 640); + tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; + out[20] = tmp_0; + tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 672); + tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; + out[21] = tmp_0; + tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 704); + tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; + out[22] = tmp_0; + tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 736); + tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; + out[23] = tmp_0; + tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 768); + tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; + out[24] = tmp_0; + tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 800); + tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; + out[25] = tmp_0; + tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 832); + tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; + out[26] = tmp_0; + tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 864); + tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; + out[27] = tmp_0; + tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 896); + tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; + out[28] = tmp_0; + tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 928); + tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; + out[29] = tmp_0; + tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); + register_0 = *(in + (0 * 32) + (i * 1) + 960); + tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; + out[30] = tmp_0; + tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); + out[31] = tmp_0; +} +__device__ __forceinline__ void unpack_32bw_32ow_32crw_1uf(const uint32_t* __restrict a_in_p, + uint32_t* __restrict a_out_p) { + [[maybe_unused]] auto out = reinterpret_cast(a_out_p); + [[maybe_unused]] auto in = reinterpret_cast(a_in_p); + [[maybe_unused]] uint32_t register_0; + [[maybe_unused]] uint32_t tmp_0; + [[maybe_unused]] uint32_t base_0 = 0ULL; + + int i = 0; // THREAD INDEX + + register_0 = *(in + (0 * 32) + (i * 1) + 0); + out[0] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 32); + out[1] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 64); + out[2] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 96); + out[3] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 128); + out[4] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 160); + out[5] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 192); + out[6] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 224); + out[7] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 256); + out[8] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 288); + out[9] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 320); + out[10] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 352); + out[11] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 384); + out[12] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 416); + out[13] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 448); + out[14] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 480); + out[15] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 512); + out[16] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 544); + out[17] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 576); + out[18] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 608); + out[19] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 640); + out[20] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 672); + out[21] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 704); + out[22] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 736); + out[23] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 768); + out[24] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 800); + out[25] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 832); + out[26] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 864); + out[27] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 896); + out[28] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 928); + out[29] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 960); + out[30] = register_0; + register_0 = *(in + (0 * 32) + (i * 1) + 992); + out[31] = register_0; +} +__device__ __forceinline__ void unpack(const uint32_t* __restrict a_in_p, uint32_t* __restrict a_out_p, uint8_t bw) { + switch (bw) { + case 0: + unpack_0bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 1: + unpack_1bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 2: + unpack_2bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 3: + unpack_3bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 4: + unpack_4bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 5: + unpack_5bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 6: + unpack_6bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 7: + unpack_7bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 8: + unpack_8bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 9: + unpack_9bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 10: + unpack_10bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 11: + unpack_11bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 12: + unpack_12bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 13: + unpack_13bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 14: + unpack_14bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 15: + unpack_15bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 16: + unpack_16bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 17: + unpack_17bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 18: + unpack_18bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 19: + unpack_19bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 20: + unpack_20bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 21: + unpack_21bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 22: + unpack_22bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 23: + unpack_23bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 24: + unpack_24bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 25: + unpack_25bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 26: + unpack_26bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 27: + unpack_27bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 28: + unpack_28bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 29: + unpack_29bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 30: + unpack_30bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 31: + unpack_31bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + case 32: + unpack_32bw_32ow_32crw_1uf(a_in_p, a_out_p); + break; + } +} + +} // namespace adjusted +} // namespace oldfls + +#endif // OLD_FLS_KERNELS_H diff --git a/galp/src/include/flsgpu/structs.cuh b/galp/src/include/flsgpu/structs.cuh new file mode 100644 index 00000000..bb40f290 --- /dev/null +++ b/galp/src/include/flsgpu/structs.cuh @@ -0,0 +1,411 @@ + +#ifndef STRUCTS_CUH +#define STRUCTS_CUH + +#include "alp.hpp" +#include "device-types.cuh" +#include "host-utils.cuh" +#include "utils.cuh" +#include +#include +#include + +namespace flsgpu { + +namespace device { + +template +struct FunctorBase { + using UINT_T = typename utils::same_width_uint::type; + + virtual __device__ __forceinline__ T operator()(const UINT_T value, [[maybe_unused]] const vi_t vector_index); +}; + +template +struct DecompressorBase { + /* Constructor, cannot be enforced + __device__ DecompressorBase(const ColumnT column, + const vi_t vector_index, const lane_t lane) + */ + + virtual void __device__ unpack_next_into(T* __restrict out); +}; + +template +struct BPColumn { + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + size_t n_vecs; + + UINT_T* packed_array; + vbw_t* bit_widths; + size_t* vector_offsets; +}; + +template +struct FFORColumn { + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + BPColumn bp; + UINT_T* bases; +}; + +template +struct ALPColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + FFORColumn ffor; + + INT_T* factors; + T* fractions; + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* counts; +}; + +template +struct ALPExtendedColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + size_t n_values; + FFORColumn ffor; + + INT_T* factors; + T* fractions; + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* offsets_counts; +}; +} // namespace device + +namespace host { + +template +struct BPColumn { + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::BPColumn; + + size_t n_values; + size_t n_packed_values; + + size_t get_n_values() const { + return n_values; + } + size_t get_n_vecs() const { + return utils::get_n_vecs_from_size(n_values); + } + + UINT_T* packed_array; + vbw_t* bit_widths; + size_t* vector_offsets; + + device::BPColumn copy_to_device() const { + const size_t branchless_extra_access_buffer = sizeof(T) * utils::get_n_lanes() * 4; + return device::BPColumn { + n_values, + get_n_vecs(), + GPUArray(n_packed_values, branchless_extra_access_buffer, packed_array).release(), + GPUArray(get_n_vecs(), bit_widths).release(), + GPUArray(get_n_vecs(), vector_offsets).release()}; + } +}; + +template +struct FFORColumn { + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::FFORColumn; + + BPColumn bp; + UINT_T* bases; + + size_t get_n_values() const { + return bp.n_values; + } + size_t get_n_vecs() const { + return bp.get_n_vecs(); + } + + device::FFORColumn copy_to_device() const { + return device::FFORColumn { + get_n_values(), bp.copy_to_device(), GPUArray(bp.get_n_vecs(), bases).release()}; + } +}; + +template +struct ALPExtendedColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::ALPExtendedColumn; + + FFORColumn ffor; + + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* offsets_counts; + + size_t compressed_size_bytes_alp_extended; + + size_t get_n_values() const { + return ffor.bp.n_values; + } + size_t get_n_vecs() const { + return ffor.bp.get_n_vecs(); + } + + double get_compression_ratio() const { + return static_cast(ffor.bp.n_values * sizeof(T)) / + static_cast(compressed_size_bytes_alp_extended); + } + + device::ALPExtendedColumn copy_to_device() const { + size_t branchless_and_prefetch_buffer = consts::MAX_UNPACK_N_VECS; + return device::ALPExtendedColumn { + get_n_values(), + ffor.copy_to_device(), + GPUArray(consts::as::FACT_ARR_COUNT, alp::Constants::FACT_ARR.data()).release(), + GPUArray(consts::as::FRAC_ARR_COUNT, alp::Constants::FRAC_ARR.data()).release(), + GPUArray(ffor.bp.get_n_vecs(), factor_indices).release(), + GPUArray(ffor.bp.get_n_vecs(), fraction_indices).release(), + n_exceptions, + GPUArray(ffor.bp.get_n_vecs(), exceptions_offsets).release(), + GPUArray(n_exceptions, branchless_and_prefetch_buffer, exceptions).release(), + GPUArray(n_exceptions, branchless_and_prefetch_buffer, positions).release(), + GPUArray(ffor.bp.get_n_vecs() * utils::get_n_lanes(), offsets_counts).release(), + }; + } +}; + +template +struct ALPColumn { + using INT_T = typename utils::same_width_int::type; + using UINT_T = typename utils::same_width_uint::type; + using DeviceColumnT = typename device::ALPColumn; + + FFORColumn ffor; + + uint8_t* factor_indices; + uint8_t* fraction_indices; + + size_t n_exceptions; + size_t* exceptions_offsets; + T* exceptions; + uint16_t* positions; + uint16_t* counts; + + size_t compressed_size_bytes_alp; + size_t compressed_size_bytes_alp_extended; + + size_t get_n_values() const { + return ffor.bp.n_values; + } + size_t get_n_vecs() const { + return ffor.bp.get_n_vecs(); + } + + double get_compression_ratio() const { + return static_cast(ffor.bp.n_values * sizeof(T)) / static_cast(compressed_size_bytes_alp); + } + + device::ALPColumn copy_to_device() const { + return device::ALPColumn { + get_n_values(), + ffor.copy_to_device(), + GPUArray(consts::as::FACT_ARR_COUNT, alp::Constants::FACT_ARR.data()).release(), + GPUArray(consts::as::FRAC_ARR_COUNT, alp::Constants::FRAC_ARR.data()).release(), + GPUArray(ffor.bp.get_n_vecs(), factor_indices).release(), + GPUArray(ffor.bp.get_n_vecs(), fraction_indices).release(), + n_exceptions, + GPUArray(ffor.bp.get_n_vecs(), exceptions_offsets).release(), + GPUArray(n_exceptions, exceptions).release(), + GPUArray(n_exceptions, positions).release(), + GPUArray(ffor.bp.get_n_vecs(), counts).release(), + }; + } + + std::tuple convert_exceptions_to_lane_divided_format() const { + constexpr auto N_LANES = utils::get_n_lanes(); + constexpr auto VALUES_PER_LANE = utils::get_values_per_lane(); + + // New exception allocations + T* out_exceptions = reinterpret_cast(malloc(sizeof(T) * n_exceptions)); + uint16_t* out_positions = reinterpret_cast(malloc(sizeof(uint16_t) * n_exceptions)); + uint16_t* out_offsets_counts = + reinterpret_cast(malloc(sizeof(uint16_t) * ffor.get_n_vecs() * N_LANES)); + + // Intermediate arrays for reordering positions and exceptions + T vec_exceptions[consts::VALUES_PER_VECTOR]; + T vec_exceptions_positions[consts::VALUES_PER_VECTOR]; + uint16_t lane_counts[N_LANES]; + + // Copies of pointers for pointer arithmetic + T* c_exceptions = exceptions; + uint16_t* c_positions = positions; + T* c_out_exceptions = out_exceptions; + uint16_t* c_out_positions = out_positions; + uint16_t* c_out_offsets_counts = out_offsets_counts; + + for (size_t vec_index {0}; vec_index < ffor.get_n_vecs(); ++vec_index) { + uint32_t vec_exception_count = counts[vec_index]; + + // Reset counts + for (size_t j {0}; j < N_LANES; ++j) { + lane_counts[j] = 0; + } + + // Split all exceptions into lanes + for (size_t exception_index {0}; exception_index < vec_exception_count; ++exception_index) { + T exception = c_exceptions[exception_index]; + uint16_t position = c_positions[exception_index]; + + uint32_t lane = position % N_LANES; + uint32_t lane_exception_count = lane_counts[lane]; + ++lane_counts[lane]; + vec_exceptions[lane * VALUES_PER_LANE + lane_exception_count] = exception; + vec_exceptions_positions[lane * VALUES_PER_LANE + lane_exception_count] = position; + } + + // Merge and concatenate all exceptions per lane into single contiguous + // array + uint32_t vec_exceptions_counter = 0; + for (size_t lane {0}; lane < N_LANES; ++lane) { + uint32_t exc_in_lane_count = lane_counts[lane]; + for (size_t exc_in_lane {0}; exc_in_lane < exc_in_lane_count; ++exc_in_lane) { + + c_out_exceptions[vec_exceptions_counter] = vec_exceptions[lane * VALUES_PER_LANE + exc_in_lane]; + c_out_positions[vec_exceptions_counter] = + vec_exceptions_positions[lane * VALUES_PER_LANE + exc_in_lane]; + ++vec_exceptions_counter; + } + + c_out_offsets_counts[lane] = (exc_in_lane_count << 10) | (vec_exceptions_counter - exc_in_lane_count); + } + + c_exceptions += vec_exception_count; + c_positions += vec_exception_count; + c_out_exceptions += vec_exception_count; + c_out_positions += vec_exception_count; + c_out_offsets_counts += utils::get_n_lanes(); + } + + return std::make_tuple(out_exceptions, out_positions, out_offsets_counts); + } + + ALPExtendedColumn create_extended_column() const { + auto [e_exceptions, e_positions, e_offsets_counts] = convert_exceptions_to_lane_divided_format(); + return ALPExtendedColumn {FFORColumn { + BPColumn { + ffor.bp.n_values, + ffor.bp.n_packed_values, + utils::copy_array(ffor.bp.packed_array, ffor.bp.n_packed_values), + utils::copy_array(ffor.bp.bit_widths, get_n_vecs()), + utils::copy_array(ffor.bp.vector_offsets, get_n_vecs()), + }, + utils::copy_array(ffor.bases, get_n_vecs()), + }, + utils::copy_array(factor_indices, get_n_vecs()), + utils::copy_array(fraction_indices, get_n_vecs()), + n_exceptions, + utils::copy_array(exceptions_offsets, get_n_vecs()), + e_exceptions, + e_positions, + e_offsets_counts, + compressed_size_bytes_alp_extended}; + } +}; + +template +void free_column(BPColumn column) { + delete[] column.packed_array; + delete[] column.bit_widths; + delete[] column.vector_offsets; +} + +template +void free_column(FFORColumn column) { + free_column(column.bp); + delete[] column.bases; +} + +template +void free_column(ALPColumn column) { + free_column(column.ffor); + delete[] column.factor_indices; + delete[] column.fraction_indices; + delete[] column.exceptions_offsets; + delete[] column.exceptions; + delete[] column.positions; + delete[] column.counts; +} + +template +void free_column(ALPExtendedColumn column) { + free_column(column.ffor); + delete[] column.factor_indices; + delete[] column.fraction_indices; + delete[] column.exceptions_offsets; + delete[] column.exceptions; + delete[] column.positions; + delete[] column.offsets_counts; +} + +// Structs that are passed to the GPU cannot contain methods, +// that is why there is a separate method for the destructors +template +void free_column(device::BPColumn column) { + free_device_pointer(column.packed_array); + free_device_pointer(column.bit_widths); + free_device_pointer(column.vector_offsets); +} + +template +void free_column(device::FFORColumn column) { + free_column(column.bp); + free_device_pointer(column.bases); +} + +template +void free_column(device::ALPColumn column) { + free_column(column.ffor); + free_device_pointer(column.factors); + free_device_pointer(column.fractions); + free_device_pointer(column.factor_indices); + free_device_pointer(column.fraction_indices); + free_device_pointer(column.exceptions_offsets); + free_device_pointer(column.exceptions); + free_device_pointer(column.positions); + free_device_pointer(column.counts); +} + +template +void free_column(device::ALPExtendedColumn column) { + free_column(column.ffor); + free_device_pointer(column.factors); + free_device_pointer(column.fractions); + free_device_pointer(column.factor_indices); + free_device_pointer(column.fraction_indices); + free_device_pointer(column.exceptions_offsets); + free_device_pointer(column.exceptions); + free_device_pointer(column.positions); + free_device_pointer(column.offsets_counts); +} + +} // namespace host +} // namespace flsgpu + +#endif // STRUCTS_CUH diff --git a/galp/src/include/flsgpu/utils.cuh b/galp/src/include/flsgpu/utils.cuh new file mode 100644 index 00000000..e44e90ba --- /dev/null +++ b/galp/src/include/flsgpu/utils.cuh @@ -0,0 +1,104 @@ +#ifndef FASTLANES_UTILS_H +#define FASTLANES_UTILS_H + +#include "consts.cuh" +#include +#include +#include +#include +#include + +namespace utils { // internal functions + // +template +struct same_width_int { + using type = typename std::conditional< + sizeof(T) == 8, + int64_t, + typename std::conditional::type>::type>::type; +}; + +template +struct same_width_uint { + using type = typename std::conditional< + sizeof(T) == 8, + uint64_t, + typename std::conditional::type>::type>::type; +}; + +template +constexpr returnT sizeof_in_bits() { + return sizeof(T) * 8; +} + +template +constexpr T min(const T& a, const T& b) { + // C++-11 and older do not offer constexpr, that is why this is added here + return a <= b ? a : b; +} + +template +constexpr T_out reinterpret_type(const T_in& in) { + // Type punning, the compiler optimizes out the memcpy + // https://cuda.godbolt.org/z/bKG89YTY7 + // https://stackoverflow.com/questions/47037104/cuda-type-punning-memcpy-vs-ub-union + // T_out out; + // T_in staged = in; + // memcpy(&out, &staged, min(sizeof(T_in), sizeof(T_out))); + // return out; + T_in staged = in; + return *reinterpret_cast(&staged); +} + +template +constexpr T h_set_first_n_bits(const int32_t count) { + return (count < sizeof_in_bits() ? static_cast((T {1} << int32_t {count}) - T {1}) : static_cast(~T {0})); +} + +template +constexpr T set_first_n_bits(const int32_t count) { + using UINT_T = typename same_width_uint::type; + static_assert(std::is_integral::value, "T must be an integer type"); + + return reinterpret_type(std::numeric_limits::max() >> + reinterpret_type(utils::sizeof_in_bits() - count)); +} + +template +constexpr int32_t get_lane_bitwidth() { + return sizeof_in_bits(); +} + +template +constexpr int32_t get_n_lanes() { + return consts::REGISTER_WIDTH / get_lane_bitwidth(); +} + +template +constexpr int32_t get_values_per_lane() { + return consts::VALUES_PER_VECTOR / get_n_lanes(); +} + +template +constexpr int32_t get_compressed_vector_size(int32_t value_bit_width) { + return (consts::VALUES_PER_VECTOR * value_bit_width) / sizeof_in_bits(); +} + +constexpr size_t get_n_vecs_from_size(const size_t size) { + return (size + consts::VALUES_PER_VECTOR - 1) / consts::VALUES_PER_VECTOR; +} + +template +T* copy_array(const T* in, const size_t n_elements) { + T* out = new T[n_elements]; + std::memcpy(out, in, sizeof(T) * n_elements); + return out; +} + +} // namespace utils + +#endif // FASTLANES_UTILS_H diff --git a/galp/src/include/generator/generate_binaries.hpp b/galp/src/include/generator/generate_binaries.hpp new file mode 100644 index 00000000..816023b9 --- /dev/null +++ b/galp/src/include/generator/generate_binaries.hpp @@ -0,0 +1,116 @@ +// ============================================================================= +// generate_binaries.hpp ― Random-data generator + directory scanner (host-only) +// +// * **Now writes little-endian** (the de-facto standard on x86/ARM). +// * Automatically swaps bytes on big-endian hosts. +// * Produces float32 + float64 binaries and can scan data/floats + data/doubles. +// * Wrapped in host-only guards so NVCC’s device pass never sees . +// ============================================================================= +#ifndef GENERATE_BINARIES_HPP +#define GENERATE_BINARIES_HPP + +#include +#include +#include +#include +#include +#include +#include + +// ────────────────────────────────────────────────────────────────────────────── +// Host-only implementation (skipped entirely for __CUDA_ARCH__) +// ────────────────────────────────────────────────────────────────────────────── +namespace bin { +namespace detail { + +// compile-time endian test (uses compiler macros, no UB in constexpr) +inline constexpr bool is_little_endian = +#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + true; +#else + false; +#endif +; + +} // namespace detail + +// ----------------------------------------------------------------------------- +// write_little_endian ― write POD value in **little-endian** order +// ----------------------------------------------------------------------------- +template +inline void write_little_endian(std::ostream& os, const T& value) { + static_assert(std::is_trivially_copyable_v, "write_little_endian requires a trivially-copyable type"); + T tmp = value; + if constexpr (!detail::is_little_endian) { // host is big-endian + auto* p = reinterpret_cast(&tmp); + std::reverse(p, p + sizeof(T)); // swap to LE + } + os.write(reinterpret_cast(&tmp), sizeof(T)); +} + +// ----------------------------------------------------------------------------- +// generate_and_write ― write TOTAL random numbers (0-100, two decimals) +// to (float32 LE) and (float64 LE) +// returns the first HEAD_COUNT values +// ----------------------------------------------------------------------------- +inline std::vector +generate_and_write(const std::string& out32, const std::string& out64, size_t total, size_t head_count) { + std::mt19937_64 rng(std::random_device {}()); + std::uniform_real_distribution dist(0.0, 100.0); + + std::vector head; + head.reserve(head_count); + + std::ofstream f32(out32, std::ios::binary); + std::ofstream f64(out64, std::ios::binary); + + for (size_t i = 0; i < total; ++i) { + double val = std::round(dist(rng) * 100.0) / 100.0; + if (head.size() < head_count) + head.push_back(val); + + write_little_endian(f32, static_cast(val)); + write_little_endian(f64, val); + } + return head; +} + +// ----------------------------------------------------------------------------- +// generate_write_and_scan ― one-stop helper: +// 1) writes /1.bin + /1.bin (little-endian) +// 2) scans both directories for every *.bin +// 3) returns first HEAD_COUNT values + sorted file lists +// ----------------------------------------------------------------------------- +struct GenResult { + std::vector head; // first N random values + std::vector float_files; // *.bin under floats_dir + std::vector double_files; // *.bin under doubles_dir +}; + +inline GenResult generate_write_and_scan(const std::filesystem::path& floats_dir, + const std::filesystem::path& doubles_dir, + size_t total, + size_t head_count) { + GenResult res; + + // (1) produce "1.bin" in each directory + res.head = generate_and_write( + (floats_dir / "random.bin").string(), (doubles_dir / "random.bin").string(), total, head_count); + + // (2) collect existing *.bin files + auto collect = [](const std::filesystem::path& dir, std::vector& out) { + if (!std::filesystem::exists(dir)) + return; + for (const auto& e : std::filesystem::directory_iterator(dir)) + if (e.is_regular_file() && e.path().extension() == ".bin") + out.emplace_back(e.path().string()); + std::sort(out.begin(), out.end()); + }; + collect(floats_dir, res.float_files); + collect(doubles_dir, res.double_files); + + return res; +} + +} // namespace bin +#endif // !__CUDA_ARCH__ diff --git a/galp/src/include/nvcomp/benchmark-compressors.cuh b/galp/src/include/nvcomp/benchmark-compressors.cuh new file mode 100644 index 00000000..a1087f31 --- /dev/null +++ b/galp/src/include/nvcomp/benchmark-compressors.cuh @@ -0,0 +1,141 @@ +#include "../engine/enums.cuh" +#include "../flsgpu/flsgpu-api.cuh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef BENCHMARK_COMPRESSORS_CUH +#define BENCHMARK_COMPRESSORS_CUH + +template +bool check_if_device_buffers_are_equal(const T* a, const T* b, const size_t n_values) { + // Convert to uint8_t as we don't want to compare floats (-nan == -nan => + // false) + thrust::device_ptr d_a(reinterpret_cast(const_cast(a))); + thrust::device_ptr d_b(reinterpret_cast(const_cast(b))); + + return thrust::equal(d_a, d_a + n_values, d_b); +} + +class CudaStopwatch { +private: + cudaEvent_t startEvent, stopEvent; + cudaStream_t stream; + float milliseconds = 0.0f; + +public: + CudaStopwatch(cudaStream_t stream = 0) + : stream(stream) { + CUDA_SAFE_CALL(cudaEventCreate(&startEvent)); + CUDA_SAFE_CALL(cudaEventCreate(&stopEvent)); + } + + ~CudaStopwatch() { + CUDA_SAFE_CALL(cudaEventDestroy(startEvent)); + CUDA_SAFE_CALL(cudaEventDestroy(stopEvent)); + } + + void start() { + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + CUDA_SAFE_CALL(cudaEventRecord(startEvent, stream)); + } + + double stop() { + CUDA_SAFE_CALL(cudaEventRecord(stopEvent, stream)); + CUDA_SAFE_CALL(cudaEventSynchronize(stopEvent)); + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + + CUDA_SAFE_CALL(cudaEventElapsedTime(&milliseconds, startEvent, stopEvent)); + return milliseconds; + } + + double get_result() { + return milliseconds; + } +}; + +struct BenchmarkResult { + bool results_match; + double execution_time_ms; + double compression_ratio; + + void log_result(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const size_t n_bytes, + const std::string data_name) const { + + const char* is_valid = results_match ? "valid" : "wrong"; + + printf("%s,%s,%s,%s,%lu,%f,%f\n", + enums_nvcomp::comparison_type_to_string(comparison_type).c_str(), + enums_nvcomp::compression_type_to_string(compression_type).c_str(), + data_name.c_str(), + is_valid, + n_bytes, + execution_time_ms, + compression_ratio); + } +}; + +template +struct is_equal_to { + T value; + is_equal_to(T value) + : value(value) { + } + __host__ __device__ bool operator()(T x) { + return x == value; + } +}; + +template +BenchmarkResult benchmark_thrust(const T* input, const size_t value_count, const T value_to_search_for); + +template +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const T* input, + const size_t value_count, + const T value_to_search_for); + +template +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const T* input, + const size_t value_count, + const T value_to_search_for); + +BenchmarkResult benchmark_thrust(const float* input, const size_t value_count, const float value_to_search_for); +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const float* input, + const size_t value_count, + const float value_to_search_for); +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const float* input, + const size_t value_count, + const float value_to_search_for); + +BenchmarkResult benchmark_thrust(const double* input, const size_t value_count, const double value_to_search_for); +BenchmarkResult benchmark_alp(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType decompressor_enum, + const double* input, + const size_t value_count, + const double value_to_search_for); +BenchmarkResult benchmark_hwc(const enums_nvcomp::ComparisonType comparison_type, + const enums_nvcomp::CompressionType compression_type, + const double* input, + const size_t value_count, + const double value_to_search_for); + +#endif // BENCHMARK_COMPRESSORS_CUH diff --git a/galp/src/include/nvcomp/nvcomp-compressors.cuh b/galp/src/include/nvcomp/nvcomp-compressors.cuh new file mode 100644 index 00000000..f5691801 --- /dev/null +++ b/galp/src/include/nvcomp/nvcomp-compressors.cuh @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifndef NVCOMP_COMPRESSORS_H +#define NVCOMP_COMPRESSORS_H + +#include "engine/enums.cuh" +#include "flsgpu/flsgpu-api.cuh" +#include "nvcomp.h" +#include "nvcomp.hpp" +#include "nvcomp/benchmark-compressors.cuh" +#include "nvcomp/deflate.h" +#include "nvcomp/deflate.hpp" +#include "nvcomp/gdeflate.h" +#include "nvcomp/gdeflate.hpp" +#include "nvcomp/lz4.hpp" +#include "nvcomp/nvcompManager.hpp" +#include "nvcomp/nvcompManagerFactory.hpp" +#include "nvcomp/zstd.h" +#include "nvcomp/zstd.hpp" + +namespace hwc { +using nvcompCompressionManager = nvcomp::PimplManager; + +nvcompCompressionManager* get_compressor_manager(const enums_nvcomp::CompressionType compression_type, + const nvcompType_t data_type = NVCOMP_TYPE_CHAR, + const size_t chunk_size = 1 << 16); + +struct CompressedBuffer { + nvcomp::CompressionConfig compression_config; + uint8_t* compressed_buffer; + size_t decompressed_size; + size_t compressed_size; + + CompressedBuffer(nvcomp::CompressionConfig compression_config) + : compression_config(compression_config) { + + CUDA_SAFE_CALL(cudaMalloc(&compressed_buffer, compression_config.max_compressed_buffer_size)); + } + + double get_compression_ratio() { + return static_cast(decompressed_size) / static_cast(compressed_size); + } + + double get_allocation_compression_ratio() { + return static_cast(compression_config.max_compressed_buffer_size) / + static_cast(compression_config.uncompressed_buffer_size); + } + + // Manual resource freeing + void free() { + CUDA_SAFE_CALL(cudaFree(compressed_buffer)); + } +}; + +struct Compressor { + nvcompCompressionManager* manager; + + Compressor(const enums_nvcomp::CompressionType compression_type) { + manager = get_compressor_manager(compression_type); + } + + CompressedBuffer compress(uint8_t* input_buffer, const size_t input_buffer_len) { + CompressedBuffer compressed_buffer(manager->configure_compression(input_buffer_len)); + manager->compress(input_buffer, compressed_buffer.compressed_buffer, compressed_buffer.compression_config); + compressed_buffer.decompressed_size = input_buffer_len; + compressed_buffer.compressed_size = manager->get_compressed_output_size(compressed_buffer.compressed_buffer); + return compressed_buffer; + } + + uint8_t* decompress(const CompressedBuffer compressed_buffer, CudaStopwatch& stopwatch) { + nvcomp::DecompressionConfig decomp_config = + manager->configure_decompression(compressed_buffer.compressed_buffer); + uint8_t* decompressed_buffer; + CUDA_SAFE_CALL(cudaMalloc(&decompressed_buffer, decomp_config.decomp_data_size)); + + stopwatch.start(); + manager->decompress(decompressed_buffer, compressed_buffer.compressed_buffer, decomp_config); + stopwatch.stop(); + + return decompressed_buffer; + } + + // Manual resource freeing + void free() { + delete manager; + } +}; + +__global__ void check_buffer_equality(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length, bool* out); + +bool compare_d_buffers(const uint8_t* buffer_a, const uint8_t* buffer_b, const size_t length); + +template +struct DummyColumn { + T* in; + size_t n_values; +}; + +template +struct DummyDecompressor : flsgpu::device::DecompressorBase { + using UINT_T = typename utils::same_width_uint::type; + flsgpu::device::BitUnpackerDummy> unpacker; + + __device__ __forceinline__ + DummyDecompressor(const DummyColumn column, const vi_t vector_index, const lane_t lane) + : unpacker(reinterpret_cast(column.in + vector_index * consts::VALUES_PER_VECTOR), + lane, + utils::sizeof_in_bits(), + flsgpu::device::BPFunctor()) { + } + + void __device__ unpack_next_into(T* __restrict out) { + unpacker.unpack_next_into(out); + } +}; + +} // namespace hwc + +#endif // NVCOMP_COMPRESSORS_H diff --git a/galp/test/CMakeLists.txt b/galp/test/CMakeLists.txt new file mode 100644 index 00000000..68560107 --- /dev/null +++ b/galp/test/CMakeLists.txt @@ -0,0 +1,18 @@ +add_executable(galp_tests + thrust_memcpy_test.cu + thrust_core_test.cu + galp_test.cu +) + + +target_link_libraries(galp_tests PRIVATE + galp_core + GTest::gtest_main + FastLanes::core +) + +apply_warnings(galp_tests) + +gtest_discover_tests(galp_tests + DISCOVERY_TIMEOUT 60 +) \ No newline at end of file diff --git a/galp/test/galp_test.cu b/galp/test/galp_test.cu new file mode 100644 index 00000000..8a8fb7cb --- /dev/null +++ b/galp/test/galp_test.cu @@ -0,0 +1,232 @@ +#include "alp/alp-bindings.cuh" +#include "data/fastlanes_data.hpp" +#include "engine/device-utils.cuh" +#include "engine/kernels.cuh" +#include "flsgpu/alp.cuh" +#include "flsgpu/structs.cuh" +#include "generator/generate_binaries.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for std::pair +#include + +template +std::vector read_file(const std::string& path) { + // Open file in binary mode, positioned at end to get its size + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error("Could not open file: " + path); + } + + // Determine file size in bytes + std::streamsize bytes = file.tellg(); + if (bytes < 0) { + throw std::runtime_error("Could not determine file size: " + path); + } + + // Ensure the file contains an integral number of T elements + if (bytes % sizeof(T) != 0) { + throw std::runtime_error("File size (" + std::to_string(bytes) + " bytes) is not a multiple of element size (" + + std::to_string(sizeof(T)) + " bytes)"); + } + + // Calculate number of elements + std::size_t count = static_cast(bytes / sizeof(T)); + + // Seek back to beginning and read all data into a vector + file.seekg(0, std::ios::beg); + std::vector data(count); + if (!file.read(reinterpret_cast(data.data()), bytes)) { + throw std::runtime_error("Error reading file: " + path); + } + + return data; +} + +// ----------------------------------------------------------------------------- +// CSV loader – light‑weight, header‑only +// ----------------------------------------------------------------------------- +template +std::vector read_csv(const std::filesystem::path& path) { + std::ifstream file(path); + if (!file) { + throw std::runtime_error("Could not open csv file: " + path.string()); + } + + std::vector data; + std::string line; + while (std::getline(file, line)) { + std::stringstream ss(line); + std::string cell; + while (std::getline(ss, cell, ',')) { + if (!cell.empty()) { + data.push_back(static_cast(std::stod(cell))); + } + } + } + return data; +} + +template +bool check_if_device_buffers_are_equal(const T* a, const T* b, const size_t n_values) { + // Convert to uint8_t as we don't want to compare floats (-nan == -nan => + // false) + thrust::device_ptr d_a(reinterpret_cast(const_cast(a))); + thrust::device_ptr d_b(reinterpret_cast(const_cast(b))); + + return thrust::equal(d_a, d_a + n_values, d_b); +} + +template +using ALPDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::StatefulALPExceptionPatcher, + flsgpu::device::ALPColumn>; + +template +using ALPExtendedDecompressor = typename flsgpu::device::ALPDecompressor< + T, + UNPACK_N_VECTORS, + flsgpu::device:: + BitUnpackerStatefulBranchless>, + flsgpu::device::PrefetchAllALPExceptionPatcher, + flsgpu::device::ALPExtendedColumn>; + +struct CLIArgs; +template +inline void test_alp(const std::filesystem::path& path) { + auto data_vec = read_file(path); + + flsgpu::host::ALPColumn column = alp::encode(data_vec.data(), data_vec.size(), true); + GPUArray d_decompression_result(data_vec.size()); + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + + flsgpu::device::ALPColumn d_column = column.copy_to_device(); + kernels::device:: + decompress_column, flsgpu::device::ALPColumn> + <<>>(d_column, d_decompression_result.get()); + + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + flsgpu::host::free_column(d_column); + + bool kernel_successful = false; + GPUArray d_input(data_vec.size(), data_vec.data()); + + kernel_successful = + check_if_device_buffers_are_equal(d_decompression_result.get(), d_input.get(), column.get_n_values()); + + EXPECT_TRUE(kernel_successful); + + flsgpu::host::free_column(column); +} + +struct CLIArgs; +template +inline void test_galp(const std::filesystem::path& path) { + auto data_vec = read_file(path); + + flsgpu::host::ALPColumn column = alp::encode(data_vec.data(), data_vec.size(), true); + flsgpu::host::ALPExtendedColumn column_extended = column.create_extended_column(); + flsgpu::device::ALPExtendedColumn d_column = column_extended.copy_to_device(); + GPUArray d_decompression_result(data_vec.size()); + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + kernels::device::decompress_column, + flsgpu::device::ALPExtendedColumn> + <<>>(d_column, d_decompression_result.get()); + + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + double compression_ratio = column_extended.get_compression_ratio(); + std::cout << "compression_ratio : " << compression_ratio << std::endl; + flsgpu::host::free_column(column_extended); + flsgpu::host::free_column(d_column); +} + +// ----------------------------------------------------------------------------- +// Exact copy of test_galp<…>, but uses read_csv instead of read_file +// ----------------------------------------------------------------------------- +template +inline void test_galp_csv(const std::filesystem::path& path) { + auto data_vec = read_csv(path); + + flsgpu::host::ALPColumn column = alp::encode(data_vec.data(), data_vec.size(), true); + flsgpu::host::ALPExtendedColumn column_extended = column.create_extended_column(); + flsgpu::device::ALPExtendedColumn d_column = column_extended.copy_to_device(); + + GPUArray d_decompression_result(data_vec.size()); + constexpr int32_t UNPACK_N_VECTORS = 1; + const ThreadblockMapping mapping(UNPACK_N_VECTORS, column.get_n_vecs()); + + kernels::device::decompress_column, + flsgpu::device::ALPExtendedColumn> + <<>>(d_column, d_decompression_result.get()); + + CUDA_SAFE_CALL(cudaDeviceSynchronize()); + double compression_ratio = column_extended.get_compression_ratio(); + std::cout << "compression_ratio : " << compression_ratio << std::endl; + + flsgpu::host::free_column(column_extended); + flsgpu::host::free_column(d_column); +} + +TEST(ALP, TEST_ALP) { + // ------------------------------------------------------------------------- + // (0) Generate new binaries + discover directories + // ------------------------------------------------------------------------- + const size_t TOTAL = 25'600 * 1'024; + const size_t HEAD = 0; + + const std::filesystem::path floats_dir = FLS_GALP_SOURCE_DIR "/data/floats"; + const std::filesystem::path doubles_dir = FLS_GALP_SOURCE_DIR "/data/doubles"; + + bin::GenResult gen = bin::generate_write_and_scan(floats_dir, doubles_dir, TOTAL, HEAD); + for (const auto& path : gen.float_files) { + test_alp(path); + } +} + +TEST(GALP, TEST_GALP) { + // ------------------------------------------------------------------------- + // (0) Generate new binaries + discover directories + // ------------------------------------------------------------------------- + const size_t TOTAL = 25'600 * 1'024; + const size_t HEAD = 0; + + const std::filesystem::path floats_dir = FLS_GALP_SOURCE_DIR "/data/floats"; + const std::filesystem::path doubles_dir = FLS_GALP_SOURCE_DIR "/data/doubles"; + + bin::GenResult gen = bin::generate_write_and_scan(floats_dir, doubles_dir, TOTAL, HEAD); + for (const auto& path : gen.float_files) { + test_galp(path); + } +} + +TEST(GALP, TEST_GALP_BY_GALP_DATASET) { + namespace fs = std::filesystem; + + for (const auto& [name, dir] : fastlanes::galp::dataset) { + for (const auto& entry : fs::directory_iterator(dir)) { + if (entry.is_regular_file() && entry.path().extension() == ".csv") { + std::cout << "Testing file: " << entry.path().string() << std::endl; + test_galp_csv(entry.path()); + } + } + } +} diff --git a/galp/test/thrust_core_test.cu b/galp/test/thrust_core_test.cu new file mode 100644 index 00000000..0789f93c --- /dev/null +++ b/galp/test/thrust_core_test.cu @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// -------------------------------------------------------------------- +// Custom functor for zip transform +// -------------------------------------------------------------------- +struct SumTuple { + __host__ __device__ int operator()(const thrust::tuple& t) const { + return thrust::get<0>(t) + thrust::get<1>(t); + } +}; + +// -------------------------------------------------------------------- +// 1. Containers & element-wise algorithms +// -------------------------------------------------------------------- +TEST(ThrustCore, TransformAdd) { + const int N = 128; + thrust::device_vector a(N); + thrust::device_vector b(N); + thrust::device_vector c(N); + + thrust::sequence(a.begin(), a.end(), 0); // 0..N-1 + thrust::fill(b.begin(), b.end(), 1); // all 1 + + thrust::transform(a.begin(), a.end(), b.begin(), c.begin(), thrust::plus()); + + thrust::host_vector h(c); + for (int i = 0; i < N; ++i) + EXPECT_EQ(h[i], i + 1); +} + +// -------------------------------------------------------------------- +// 2. Reductions & scans +// -------------------------------------------------------------------- +TEST(ThrustCore, ReduceAndInclusiveScan) { + const int N = 256; + thrust::device_vector d(N); + thrust::sequence(d.begin(), d.end(), 1); // 1..N + + int gpu_sum = thrust::reduce(d.begin(), d.end()); + int ref_sum = N * (N + 1) / 2; + EXPECT_EQ(gpu_sum, ref_sum); + + thrust::inclusive_scan(d.begin(), d.end(), d.begin()); + + int last = d.back(); + EXPECT_EQ(last, ref_sum); +} + +// -------------------------------------------------------------------- +// 3. Sorting & uniquing +// -------------------------------------------------------------------- +TEST(ThrustCore, SortAndUnique) { + std::vector temp = {5, 1, 4, 4, 3, 2, 1}; + thrust::host_vector h(temp.begin(), temp.end()); + thrust::device_vector d = h; + + thrust::sort(d.begin(), d.end()); + auto new_end = thrust::unique(d.begin(), d.end()); + + EXPECT_EQ(new_end - d.begin(), 5); + + thrust::host_vector out(d.begin(), new_end); + std::vector ref = {1, 2, 3, 4, 5}; + EXPECT_TRUE(std::equal(out.begin(), out.end(), ref.begin())); +} + +// -------------------------------------------------------------------- +// 4. Zip iterator, counting iterator & transform iterator +// -------------------------------------------------------------------- +TEST(ThrustCore, CountingAndZipIter) { + const int N = 64; + auto first = thrust::make_counting_iterator(0); + auto second = thrust::make_counting_iterator(0); + + thrust::device_vector d(N); + + thrust::transform(thrust::make_zip_iterator(thrust::make_tuple(first, second)), + thrust::make_zip_iterator(thrust::make_tuple(first + N, second + N)), + d.begin(), + SumTuple()); + + thrust::host_vector h = d; + for (int i = 0; i < N; ++i) + EXPECT_EQ(h[i], i + i); +} + +// -------------------------------------------------------------------- +// 5. Set difference (requires sorted ranges) +// -------------------------------------------------------------------- +TEST(ThrustCore, SetDifference) { + std::vector tempA = {0, 1, 2, 3, 4, 5}; + std::vector tempB = {1, 3, 5}; + + thrust::device_vector A(tempA.begin(), tempA.end()); + thrust::device_vector B(tempB.begin(), tempB.end()); + thrust::device_vector diff(A.size()); + + auto end = thrust::set_difference(A.begin(), A.end(), B.begin(), B.end(), diff.begin()); + + thrust::host_vector h(diff.begin(), end); + std::vector ref = {0, 2, 4}; + EXPECT_TRUE(std::equal(h.begin(), h.end(), ref.begin())); +} diff --git a/galp/test/thrust_memcpy_test.cu b/galp/test/thrust_memcpy_test.cu new file mode 100644 index 00000000..11e733b4 --- /dev/null +++ b/galp/test/thrust_memcpy_test.cu @@ -0,0 +1,25 @@ +// test/test_thrust_memcpy.cu + +#include +#include +#include +#include +#include + +TEST(ThrustMemcpy, HostToDeviceAndBack) { + constexpr std::size_t N = 1 << 12; // 4096 ints + + // 1) Host data: 0,1,2,…,N-1 + thrust::host_vector h_src(N); + thrust::sequence(h_src.begin(), h_src.end()); + + // 2) Copy to device (H→D constructor) + thrust::device_vector d_buf = h_src; + + // 3) Copy back to host (D→H constructor) + thrust::host_vector h_dst = d_buf; + + // 4) Verify equality with a single call + bool same = thrust::equal(h_src.begin(), h_src.end(), h_dst.begin()); + EXPECT_TRUE(same) << "Data changed after GPU round-trip"; +} diff --git a/gpu/CMakeLists.txt b/gpu/CMakeLists.txt deleted file mode 100644 index 8b137891..00000000 --- a/gpu/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/gpu/cuda/CMakeLists.txt b/gpu/cuda/CMakeLists.txt deleted file mode 100644 index 8b137891..00000000 --- a/gpu/cuda/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/mk/benchmark.mk b/mk/benchmark.mk new file mode 100644 index 00000000..d390eba3 --- /dev/null +++ b/mk/benchmark.mk @@ -0,0 +1,34 @@ +# mk/benchmark.mk +# ──────────────── +# Runs benchmark/result/galp/plot.py from *inside* its own folder, +# so the script can see the neighbouring patch.csv file. + +# ── Virtual‑env (absolute so it still works after any cd) +VENV_DIR := $(abspath ../.venv) + +PYTHON3 := $(VENV_DIR)/bin/python3 +PYTHON := $(if $(wildcard $(PYTHON3)),$(PYTHON3),$(VENV_DIR)/bin/python) +PIP := $(PYTHON) -m pip + +# ── Script location (relative to project root) +PY_SCRIPT := benchmark/result/galp/plot.py +PY_SCRIPT_DIR := $(dir $(PY_SCRIPT)) # benchmark/result/galp/ +PY_SCRIPT_NAME := $(notdir $(PY_SCRIPT)) # plot.py + +.PHONY: venv plot clean-venv + +venv: + @echo "Creating virtualenv in $(VENV_DIR)…" + @test -d $(VENV_DIR) || python3 -m venv $(VENV_DIR) + @echo "Installing deps into venv…" + @$(PIP) install --upgrade pip pandas matplotlib + +plot: venv + @echo "Plotting from patch.csv…" + # 1) cd to project root 2) cd into script’s folder 3) run script + @cd .. && cd $(PY_SCRIPT_DIR) && $(PYTHON) $(PY_SCRIPT_NAME) + @echo "Done." + +clean-venv: + @rm -rf $(VENV_DIR) + @echo "Removed venv." diff --git a/mk/format.mk b/mk/format.mk index 9c937499..d3d0b064 100644 --- a/mk/format.mk +++ b/mk/format.mk @@ -1,6 +1,6 @@ IMAGE := ghcr.io/azimafroozeh/clang-format-python/clang-format-python:14 UBUNTU_IMAGE := ubuntu:22.04 -DIRS := examples src benchmark test data/include python +DIRS := examples src benchmark test data/include python cuda galp EXCLUDES := --exclude src/include/fls/json/nlohmann \ --exclude src/include/flatbuffers \ --exclude src/primitives \ @@ -14,8 +14,10 @@ DOCKER := docker run --rm -v "$$(pwd)":/app -w /app # the actual clang-format invocation FMT_CMD := python3 scripts/run-clang-format.py \ -r $(DIRS) \ + --extensions='c,h,C,H,cpp,hpp,cc,hh,cxx,hxx,cu,cuh' \ $(EXCLUDES) + .PHONY: format clang-format format-check rust-format-check clang-format: diff --git a/scripts/generate_synthetic_data.py b/scripts/generate_synthetic_data.py index 43277d95..49e2b369 100644 --- a/scripts/generate_synthetic_data.py +++ b/scripts/generate_synthetic_data.py @@ -18,7 +18,7 @@ from generator_helpers.boolean_generator import * from generator_helpers.i08_generator import * from generator_helpers.u08_generator import * - +from generator_helpers.galp_generator import * # --------------------------- # Faker Providers @@ -622,6 +622,7 @@ def main(): generate_any_value_count() generate_subnormals() generate_fsst_test() + fls_galp() logging.info("✅ All data files generated successfully.") # ← NEW diff --git a/scripts/generator_helpers/galp_generator.py b/scripts/generator_helpers/galp_generator.py new file mode 100644 index 00000000..f6606eaf --- /dev/null +++ b/scripts/generator_helpers/galp_generator.py @@ -0,0 +1,76 @@ +""" +Module: scripts/generate_helpers/galp_generator.py +Description: For three fields—one random two‑decimal GALP, one constant 1.00, + and one POSITIVE_INF—writes each as its own single‑column CSV + + schema.json, in both ROW_GROUP_SIZE and VEC_SIZE variants, + and for both FLOAT and DOUBLE schema types. +""" + +import random +from decimal import Decimal, ROUND_HALF_UP +from pathlib import Path +import json + +from .write_helpers import write_csv +from .common import ROW_GROUP_SIZE, VEC_SIZE + + +def _gen_galp(_faker, row_id): + """Random number in [0.0,100.0], two decimals.""" + raw = random.uniform(0.0, 100.0) + return [ + float( + Decimal(str(raw)) + .quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) + ) + ] + + +def _gen_constant(_faker, row_id): + """Constant = 1.00.""" + return [1.00] + + +def _gen_inf(_faker, row_id): + """Constant = +infinity.""" + return [float("inf")] + + +def fls_galp(): + """ + For each of three fields—GALP, CONSTANT, POSITIVE_INF—and for each precision + (FLOAT vs DOUBLE), write: + + data/generated/galp///single_column/generated.csv (ROW_GROUP_SIZE rows) + data/generated/galp///single_column/schema.json + + data/generated/galp///one_vector/generated.csv (VEC_SIZE rows) + data/generated/galp///one_vector/schema.json + """ + base_dir = Path.cwd() / "../data" / "generated" / "galp" + + # the three fields we want to generate + tasks = [ + ("galp", "SYNTHETIC_DATA_GALP", _gen_galp), + ("constant", "SYNTHETIC_DATA_CONSTANT", _gen_constant), + ("positive_inf", "SYNTHETIC_DATA_POSITIVE_INF", _gen_inf), + ] + + # emit both FLOAT and DOUBLE versions + precisions = [ + ("float", "FLOAT"), + ("double", "DOUBLE"), + ] + + for prec_key, prec_type in precisions: + for field_key, column_name, gen_fn in tasks: + # 1) Row‑group (single_column) + dir_rg = base_dir / prec_key / field_key / "single_column" + write_csv(dir_rg, gen_fn, ROW_GROUP_SIZE) + schema = {"columns": [{"name": column_name, "type": prec_type}]} + (dir_rg / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") + + # 2) One‑vector version + dir_ov = base_dir / prec_key / field_key / "one_vector" + write_csv(dir_ov, gen_fn, VEC_SIZE) + (dir_ov / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") diff --git a/src/connection.cpp b/src/connection.cpp index 974e04f0..9f5b3708 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -54,18 +54,23 @@ up Connection::read_fls(const path& file_path) { return make_unique(file_path, *this); } -void prepare_rowgroup(Rowgroup& rowgroup) { - - // could be combined +void prepare_rowgroup(Rowgroup& rowgroup, const Config& config) { + // init rowgroup.Init(); - rowgroup.Cast(); + + // Only cast if schema wasn’t forced + const bool shouldCast = !config.is_forced_schema && !config.is_forced_schema_pool; + if (shouldCast) { + rowgroup.Cast(); + } + rowgroup.Finalize(); rowgroup.GetStatistics(); } void Connection::prepare_table() const { for (auto& rowgroup : m_table->m_rowgroups) { - prepare_rowgroup(*rowgroup); + prepare_rowgroup(*rowgroup, *m_config); } } diff --git a/src/cor/compression_plan_history.cpp b/src/cor/compression_plan_history.cpp deleted file mode 100644 index fc0b2ff0..00000000 --- a/src/cor/compression_plan_history.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "fls/cor/compression_plan_history.hpp" -#include "fls/cor/compression_plan.hpp" -#include - -namespace fast_lanes { -void History::Save(sp debug_sp) { /* NOLINT*/ - m_plan_sp_vec.push_back(debug_sp); -} - -void History::Last(std::ostream& out) { - m_plan_sp_vec.back()->ToCsv(out); -} -void History::ToTable(TxtTab& txt_tab) { - m_plan_sp_vec.back()->ToTable(txt_tab); -} -} // namespace fast_lanes diff --git a/src/cor/cpu.cpp b/src/cor/cpu.cpp deleted file mode 100644 index 3e2b39da..00000000 --- a/src/cor/cpu.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "fls/cor/cpu.hpp" -namespace fast_lanes { -} // namespace fast_lanes diff --git a/src/cor/lyt/col.cpp b/src/cor/lyt/col.cpp deleted file mode 100644 index 4274c8f7..00000000 --- a/src/cor/lyt/col.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include "fls/cor/lyt/col.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/io/external_memory.hpp" - -namespace fastlanes { -template -Col::Col() - : m_count {0} { - m_buffer.resize(CFG::ROW_GROUP_SIZE); -} - -template -T* Col::operator[](n_t i) { - if constexpr (std::is_same()) { - FLS_ABORT("Not Supported") - } else { - FLS_ASSERT(i >= 0, "negative index.", " "); - FLS_ASSERT(m_count != 0, "0 size.", " "); - FLS_ASSERT(i < m_count, "Out of range index.", " "); - - return &(m_buffer[i]); - } - return nullptr; -} - -template -void Col::Share(T* data, n_t c) { -} - -template -void Col::Insert(T* data, n_t c) { - if constexpr (std::is_same()) { - FLS_ABORT("Not Supported") - } else { - FLS_ASSERT(m_count + c <= CFG::ROW_GROUP_SIZE, " ", "over_capacity"); - - ExternalMemory::Copy(m_buffer.data() + m_count, data, c * sizeof(T)); - m_count += c; - } -} - -template -T* Col::Data() { - // return m_buffer.data(); - return nullptr; -} - -template -n_t Col::Count() const { - return m_count; -} - -template -byte_c Col::Size() const { - return byte_c {sizeof(T) * m_buffer.size()}; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(Col) -} // namespace fastlanes diff --git a/src/cor/lyt/dic/CMakeLists.txt b/src/cor/lyt/dic/CMakeLists.txt deleted file mode 100644 index 726579b2..00000000 --- a/src/cor/lyt/dic/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_library(dic - OBJECT - dic.cpp - dic_fac.cpp -) - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -if (ENABLE_IWYU) - set_property(TARGET dic PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () diff --git a/src/cor/lyt/dic/dic.cpp b/src/cor/lyt/dic/dic.cpp deleted file mode 100644 index a8ee486a..00000000 --- a/src/cor/lyt/dic/dic.cpp +++ /dev/null @@ -1,275 +0,0 @@ -#include "fls/cor/lyt/dic/dic.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/eng/hzl_cmp_fac.hpp" -#include "fls/cor/eng/hzl_de_cmp_fac.hpp" -#include "fls/cor/prm/fsst/fsst.h" -#include "fls/logger/logger.hpp" -#include -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -DictHdrT DictHdrT::load(const uint8_t* p) { - DictHdrT dict_hdr; - std::memcpy(&dict_hdr, p, sizeof(dict_hdr)); - return dict_hdr; -} - -Dic::Dic(AnalyzeState& stt) - : data_buf {stt.data_buf} - , offs_buf {stt.offs_buf} - , compressed_data_buf {stt.compressed_data_buf} - , compressed_offs_buf {stt.compressed_offs_buf} { -} - -Dic::Dic(DecompressState& stt) - : data_buf {stt.data_buf} - , offs_buf {stt.offs_buf} - , compressed_data_buf {stt.decompressed_data_buf} - , compressed_offs_buf {stt.decompressed_offs_buf} { -} - -Dic::~Dic() = default; - -template -TDic::TDic(AnalyzeState& stt) - : Dic {stt} { - hzl_cmp_sp = HzlCmpFac::Create(stt, PageT::DICT_PAGE); - hzl_cmp_sp->Reset(); -}; - -template -TDic::TDic(DecompressState& stt) - : Dic {stt} { - hzl_de_cmp_sp = HzlDeCmpFac::Create(stt.exp_t, 0, PageT::DICT_PAGE); -}; - -template -n_t TDic::size() { - auto ttl_sz {sizeof(DictHdrT)}; - ttl_sz += compressed_offs_buf.length(); - - if constexpr (std::is_same() || std::is_same_v) { - FLS_ASSERT_ZERO(offs_buf.length()) - // always compress - ttl_sz += compressed_data_buf.length(); - } else if constexpr (std::is_same()) { - ttl_sz += compressed_data_buf.length(); - ttl_sz += FSST_MAXHEADER; - } else { - FLS_ABORT("NOT SUPPORTED") - } - - return ttl_sz; -} - -template -void TDic::Compress(CompressState& stt) { - stt.c = static_cast(map.size() + exc_map.size()); - - if constexpr (std::is_same_v) { - serialize(); - hzl_cmp_sp->FFOR(data_buf, compressed_data_buf, stt); - } else if constexpr (std::is_same_v) { - serialize(); - hzl_cmp_sp->ALP(data_buf, compressed_data_buf, stt); - } else if constexpr (std::is_same_v) { - serialize(); - hzl_cmp_sp->FSST(data_buf, offs_buf, compressed_data_buf, compressed_offs_buf, stt); - - } else { - FLS_ABORT("NOT SUPPORTED") - } -} - -template -void TDic::Write(Buf& buf, CompressState& stt) { - /* HDR*/ - DictHdrT dict_hdr = {ExpT::DICT, 0, 0, 0, static_cast(map.size() + exc_map.size())}; - - buf.UnsafeAppend(&dict_hdr, sizeof(dict_hdr)); - - if constexpr (std::is_same_v || std::is_same_v) { - FLS_ASSERT_ZERO(offs_buf.length()) - buf.UnsafeAppend(compressed_data_buf.data(), compressed_data_buf.length()); - - } else if constexpr (std::is_same_v) { - /* FSST */ - fsst_export(static_cast(stt.fsst_encoder), buf.end()); - buf.Advance(FSST_MAXHEADER); - - /* Offsets */ - buf.UnsafeAppend(compressed_offs_buf.data(), compressed_offs_buf.length()); - - /* Data */ - buf.UnsafeAppend(compressed_data_buf.data(), compressed_data_buf.length()); - } else { - FLS_ABORT("NOT SUPPORTED") - } -} - -template -void TDic::repetition_to_index() { - // [fixme] - FLS_PLOG_KEY_VALUE("fixme", "unordered_map has been used") - idx_t idx = 0; - for (auto& it : map) { - auto& c = it.second; - c = idx; - idx += 1; - } - - for (auto& it : exc_map) { - auto& c = it.second; - c = idx; - idx += 1; - } -} - -template -void TDic::Decompress(const uint8_t* a_src_p, DecompressState& stt) { - - auto* src_p = const_cast(a_src_p); - /* HDR*/ - DictHdrT dict_hdr; - std::memcpy(&dict_hdr, src_p, sizeof(dict_hdr)); - src_p += sizeof(dict_hdr); - - FLS_ASSERT_NOT_ZERO(dict_hdr.entry_c) - - if constexpr (std::is_same_v) { - hzl_de_cmp_sp->UNFFOR(const_cast(src_p), data_buf); - } - /**/ - else if constexpr (std::is_same_v) { - hzl_de_cmp_sp->UNFSST(const_cast(src_p), offs_buf, data_buf, dict_hdr.entry_c); - } - /**/ - else if constexpr (std::is_same_v) { - std::memcpy(data_buf.mutable_data(), src_p, dict_hdr.entry_c * sizeof(PT)); - // hzl_de_cmp_sp->UNALP(const_cast(src_p), data_buf); TODO - } - /**/ - else { - FLS_ABORT("NOT IMPLEMENTED") - throw std::runtime_error("NOT IMPLEMENTED!"); - } -} - -template -up TDic::get_idx_vec(const ofs_t* offset_arr, const void* a_byte_arr) { - auto idx_res_vec = std::make_unique>(); - auto last_idx = map.size() + exc_map.size(); - - if constexpr (std::is_same_v || std::is_same_v) { - const auto* in_arr = reinterpret_cast(a_byte_arr); - for (n_t i {0}; i < vec_n_tup(); ++i) { - auto val = in_arr[i]; - auto it = map.find(val); - if (it == map.end()) { - auto sec_it = exc_map.find(val); - if (sec_it == exc_map.end()) { - exc_map.emplace(val, last_idx); - idx_res_vec->push_back(last_idx); - last_idx++; - } else { - idx_res_vec->push_back(sec_it->second); - } - } else { - idx_res_vec->push_back(it->second); - } - } - return idx_res_vec; - } else if constexpr (std::is_same()) { - auto byte_arr = reinterpret_cast(a_byte_arr); - - idx_t exc_idx = map.size(); - for (ofs_t i {0}, start_off {0}, end_off {0}; i < vec_n_tup(); ++i, start_off = end_off) { - end_off = offset_arr[i]; - std::string next_string(&byte_arr[start_off], &byte_arr[end_off]); - auto it = map.find(next_string); - if (it == map.end()) { - idx_res_vec->push_back(exc_idx); - exc_idx++; - } else { - idx_res_vec->push_back(it->second); - } - } - return idx_res_vec; - } else { - FLS_ABORT("NOT SUPPORTED!") - return nullptr; - } -} - -template -bool sort_val_ascending(std::pair& a, std::pair& b) { - return a.second < b.second; -} - -template -void TDic::serialize() { - FLS_ASSERT_FALSE(map.empty()) - - if constexpr (std::is_same_v || std::is_same_v) { - auto* val_arr {reinterpret_cast(data_buf.mutable_data())}; - - // primary map - for (auto& it : map) { - auto& val = it.first; - auto& key = it.second; - val_arr[key] = val; - } - - // secondary map - for (auto& it : exc_map) { - auto& val = it.first; - auto& key = it.second; - val_arr[key] = val; - } - n_t ttl_sz = (map.size() + exc_map.size()) * sizeof(PT); - data_buf.Advance(ttl_sz); - } - /**/ - else if constexpr (std::is_same()) { - /* INDEX1_OFF_ARR*/ - uint32_t cur_ofs {0UL}; - - // [fixme] not optimal - // sort by index - std::vector> elems(map.begin(), map.end()); - std::sort(elems.begin(), elems.end(), sort_val_ascending); - - for (auto& it : elems) { - auto& str = it.first; - FLS_ASSERT_LE(cur_ofs, data_buf.Capacity()) // todo [buf] - - offs_buf.UnsafeAppend(&cur_ofs, sizeof(ofs_t)); - data_buf.UnsafeAppend(str.data(), str.size()); - - cur_ofs += str.size(); - } - - offs_buf.UnsafeAppend(&cur_ofs, sizeof(ofs_t)); - - // [fixme] add exc_map - } - /**/ - else { - FLS_ABORT("NOT SUPPORTED!") - } -} - -template -TDic::~TDic() = default; - -/*--------------------------------------------------------------------------------------------------------------------*\ -+ Specialization -\*--------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(TDic) - -} // namespace fastlanes diff --git a/src/cor/lyt/dic/dic_fac.cpp b/src/cor/lyt/dic/dic_fac.cpp deleted file mode 100644 index 984a15ab..00000000 --- a/src/cor/lyt/dic/dic_fac.cpp +++ /dev/null @@ -1,581 +0,0 @@ -#include "fls/cor/lyt/dic/dic_fac.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/lyt/dic/dic.hpp" -#include "fls/utl/random.hpp" -#include "fls/utl/util.hpp" -#include -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -enum class DictSortStrategy : uint8_t { // nolint - SORT_BY_INSERTION_ORDER = 0, - SORT_BY_FREQUENCY = 1, - SORT_BY_VALUE = 2, -}; - -constexpr DictSortStrategy get_strategy(ExpT exp_t) { - switch (exp_t) { - case ExpT::INVALID: - case ExpT::UNDECIDED: - case ExpT::UNCOMPRESSED: - case ExpT::BYTE_ARR: - case ExpT::FFOR: - case ExpT::DELTA: - case ExpT::RLE: - case ExpT::FSST: - case ExpT::DICT: - return DictSortStrategy::SORT_BY_INSERTION_ORDER; - case ExpT::DICT_RLE: - return DictSortStrategy::SORT_BY_INSERTION_ORDER; - case ExpT::DICT_FRQ: - return DictSortStrategy::SORT_BY_FREQUENCY; - case ExpT::DICT_VAL: - return DictSortStrategy::SORT_BY_VALUE; - // TODO [FIXME] - default: - return DictSortStrategy::SORT_BY_INSERTION_ORDER; - } -} - -/* Flip a map*/ -template -static std::multimap invert(std::unordered_map& map) { - if constexpr (std::is_same()) { - std::multimap multi_map; - - std::unordered_map::iterator it; - for (it = map.begin(); it != map.end(); it++) { - multi_map.insert(std::make_pair(it->second, it->first)); - } - - return multi_map; - } else if constexpr (std::is_same()) { - std::multimap multi_map; - - std::unordered_map::iterator it; - for (it = map.begin(); it != map.end(); it++) { - multi_map.insert(std::make_pair(it->second, it->first)); - } - - return multi_map; - } -} - -template -up> dict_fac::adaptive_create(const PT* data_p, n_t n, AnalyzeState& stt) { - FLS_ASSERT_CORRECT_N(n) - - const n_t vec_c {n / CFG::VEC_TUP_C}; - n_t analyzed_vec_c {0}; - if (vec_c == 0) { - return nullptr; - }; /* No dict for tuple-c < 1024 */ - - /* Init. */ - //[fixme] move to stt - vector is_processed; - is_processed.reserve(vec_c); - - n_t result {0}; - - /* base. */ - auto base_dict = dict_fac::m_create_from(data_p, 0, stt); - analyzed_vec_c++; - if (vec_c == 1) { - base_dict->repetition_to_index(); - return base_dict; - } - - /* next. */ - auto next_dict = dict_fac::m_create_random(data_p, vec_c, is_processed, stt); - analyzed_vec_c++; - - result = dict_fac::m_compare(base_dict->map, next_dict->map); - dict_fac::m_combine(base_dict->map, next_dict->map); - - /* Adaptive analyzing*/ - while (result < CFG::DIC::THRESHOLD) { - FLS_ASSERT_LE(analyzed_vec_c, vec_c) - - if (analyzed_vec_c == vec_c) { - break; - } - next_dict = dict_fac::m_create_random(data_p, vec_c, is_processed, stt); - - /* Check the dic_page limit. */ - if (base_dict->size() + next_dict->size() > CFG::DIC::DIC_PAGE_SZ) { - break; - } - - result = dict_fac::m_compare(base_dict->map, next_dict->map); - dict_fac::m_combine(base_dict->map, next_dict->map); - } - - base_dict->repetition_to_index(); - return base_dict; -} - -template -up> -dict_fac::m_create_random(const PT* data_p, n_t possible_sample_c, vector& is_processed, AnalyzeState& stt) { - auto next_batch_num = random::next_vec_num(possible_sample_c, is_processed); - return m_create_from(data_p, next_batch_num, stt); -} - -template -n_t dict_fac::m_compare(dic_t& r, dic_t& l) { - if constexpr (std::is_same()) { - n_t total {0}; - - std::unordered_map::iterator r_it; - - /* How many tuples are covered by r dic?*/ - for (r_it = r.begin(); r_it != r.end(); r_it++) { - auto l_it = l.find(r_it->first); - - if (l_it != l.end()) { - total += l_it->second; - } - } - - return total; - } else if constexpr (std::is_same()) { - n_t total {0}; - - std::unordered_map::iterator r_it; - - /* How many tuples are covered by r dic?*/ - for (r_it = r.begin(); r_it != r.end(); r_it++) { - auto l_it = l.find(r_it->first); - - if (l_it != l.end()) { - total += l_it->second; - } - } - - return total; - } else { - FLS_ABORT("NOT SUPPORTED!") - return 0; - } -} - -template -up> dict_fac::m_create_from(const PT* data_p, n_t sample_num, AnalyzeState& stt) { - if constexpr (std::is_same()) { - auto dic = std::make_unique>(stt); - - /* Create Dic */ - for (size_t i = 0; i < CFG::VEC_TUP_C; ++i) { - auto next_val = data_p[i]; - if (dic->map.find(next_val) == dic->map.end()) { - dic->map.emplace(next_val, 1); - } else { - dic->map[next_val]++; - } - } - - /* Prune the 10 percent undesired part. */ - std::multimap flipped_map = invert(dic->map); - n_t sum {0}; - - auto it = flipped_map.end(); - it--; - for (; it != flipped_map.begin();) { - sum += it->first; - --it; - if (sum >= CFG::DIC::THRESHOLD) { - /* Remove elements from now on. */ - dic->map.erase(dic->map.find(it->second)); - } - } - - return dic; - - } else if constexpr (std::is_same()) { - auto dic_up = std::make_unique>(stt); - - /* Create Dic */ - for (size_t i = 0; i < CFG::VEC_TUP_C; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - } - - /* Prune the 10 percent undesired part. */ - std::multimap flipped_map = invert(dic_up->map); - n_t visited {0}; - - for (auto it = flipped_map.end(); it != flipped_map.begin(); --it) { - visited += it->first; - if (visited >= CFG::DIC::THRESHOLD) { - /* Remove element from now on. */ - dic_up->map.erase(dic_up->map.find(it->second)); - } - } - - return dic_up; - } else { - FLS_ABORT("NOT SUPPORTED!") - return nullptr; - } -} - -template -void dict_fac::m_combine(dic_t& left, dic_t& right) { - if constexpr (std::is_same()) { - for (auto& r_it : right) { - auto l_it = left.find(r_it.first); - if (l_it != left.end()) { - left.emplace(r_it.first, r_it.second); - } else { - left[r_it.first] += r_it.second; - } - } - } - - else { - FLS_ABORT("NOT SUPPORTED!") - } -} - -template -bool sort_val_ascending(std::pair& a, std::pair& b) { - return a.second < b.second; -} - -template -bool sort_val_descending(std::pair& a, std::pair& b) { - return a.second > b.second; -} - -template -bool sort_key_ascending(std::pair& a, std::pair& b) { - return a.first < b.first; -} - -template -bool sort_key_descending(std::pair& a, std::pair& b) { - return a.first > b.first; -} - -template -up> dict_fac::perfect_create(const PT* data_p, n_t n, AnalyzeState& stt) { - if constexpr (std::is_same_v || std::is_same_v) { - auto dic_up = std::make_unique>(stt); - std::vector> elems; - auto strategy = get_strategy(stt.m_exp_t); - - switch (strategy) { - case DictSortStrategy::SORT_BY_INSERTION_ORDER: { - /* Create Dic */ - auto idx = 0; - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, idx++); - } - }; - - // pair of value - idx - elems = {dic_up->map.begin(), dic_up->map.end()}; - - // sort based on frequency! - std::sort(elems.begin(), elems.end(), sort_val_ascending); - } break; - case DictSortStrategy::SORT_BY_FREQUENCY: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - }; - - // pair of value - frequency - elems = {dic_up->map.begin(), dic_up->map.end()}; - - // sort based on frequency! - std::sort(elems.begin(), elems.end(), sort_val_descending); - - } break; - case DictSortStrategy::SORT_BY_VALUE: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - }; - - // pair of value - frequency - elems = {dic_up->map.begin(), dic_up->map.end()}; - - // sort based on frequency! - std::sort(elems.begin(), elems.end(), sort_key_ascending); - } - } - - dic_up->map.clear(); - - n_t idx {0}; - for (auto& pair : elems) { - dic_up->map.emplace(pair.first, idx); - idx++; - } - - return dic_up; - } else if constexpr (std::is_same()) { - auto dic_up = std::make_unique>(stt); - auto strategy = get_strategy(stt.m_exp_t); - std::vector> elems; - - switch (strategy) { - case DictSortStrategy::SORT_BY_INSERTION_ORDER: { - /* Create Dic */ - auto idx = 0; - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, idx++); - } - }; - - elems = {dic_up->map.begin(), dic_up->map.end()}; - std::sort(elems.begin(), elems.end(), sort_val_descending); - } break; - case DictSortStrategy::SORT_BY_FREQUENCY: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - } - - elems = {dic_up->map.begin(), dic_up->map.end()}; - std::sort(elems.begin(), elems.end(), sort_val_descending); - - } break; - case DictSortStrategy::SORT_BY_VALUE: { - /* Create Dic */ - for (size_t i = 0; i < n; ++i) { - auto next_val = data_p[i]; - if (dic_up->map.find(next_val) == dic_up->map.end()) { - dic_up->map.emplace(next_val, 1); - } else { - dic_up->map[next_val]++; - } - }; - - elems = {dic_up->map.begin(), dic_up->map.end()}; - - std::sort(elems.begin(), elems.end(), sort_key_ascending); - } - } - - dic_up->map.clear(); - - n_t idx {0}; - for (auto& pair : elems) { - dic_up->map.emplace(pair.first, idx); - idx++; - } - - return dic_up; - } else { - FLS_ABORT("NOT SUPPORTED!") - return nullptr; - } -} - -template -bsz_t dict_fac::multi_dic_create(const std::vector& src_col, - AnalyzeState& src_stt, - const std::vector& tgt_col, - AnalyzeState& tgt_stt) { - bsz_t ttl_sz {0}; - - if constexpr (std::is_same_v) { - auto src_dic = *dynamic_cast*>(src_stt.dic_up.get()); - auto tgt_dic = *dynamic_cast*>(tgt_stt.dic_up.get()); - - FLS_ABORT("HERE") - - } else if constexpr (std::is_same_v) { - } else { - } - - return ttl_sz; -} - -template -bsz_t dict_fac::multi_dic_create(const std::vector& src_col, - AnalyzeState& src_stt, - const std::vector& tgt_col, - AnalyzeState& tgt_stt) { - bsz_t ttl_sz {0}; - - FLS_ABORT("HERE") - if constexpr (std::is_same_v) { - - } else if constexpr (std::is_same_v) { - } else { - } - - return ttl_sz; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col, const std::vector& tgt_col) { - FLS_ASSERT_EQUALITY(src_col.size(), tgt_col.size()) - - bsz_t ttl_size {0}; - - std::unordered_map> new_dic; - - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - const auto& tgt_val = tgt_col[idx]; - - if (new_dic.find(src_val) == new_dic.end()) { - new_dic.insert({src_val, {}}); - } else { - new_dic[src_val].insert(tgt_val); - } - } - - sz_t max_sz {0}; - - for (const auto& map : new_dic) { - auto map_size = map.second.size(); - if (max_sz < map.second.size()) { - max_sz = map_size; - } - - for (const auto& val : map.second) { - ttl_size += val.size() + sizeof(ofs_t); - } - } - - auto bits_per_value = RANGE_BIT(max_sz); - auto ttl_bytes = src_col.size() * bits_per_value / 8; - return ttl_size + ttl_bytes; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col) { - - bsz_t ttl_size {0}; - - std::set dic; - - if constexpr (!std::is_same_v) { - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - if (dic.find(src_val) == dic.end()) { - dic.insert(src_val); - } - } - } else { - FLS_ABORT("NOT IMPLEMENTED TYPE") - } - - if constexpr (std::is_same_v) { - ttl_size += dic.size() * (sizeof(PT) + sizeof(ofs_t)); - } else if constexpr (std::is_same_v) { - for (const auto& val : dic) { - ttl_size += val.size() + sizeof(ofs_t); - } - } else { - FLS_ABORT("NOT IMPLEMENTED TYPE") - } - - return ttl_size; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col, const std::vector& tgt_col) { - FLS_ASSERT_EQUALITY(src_col.size(), tgt_col.size()) - - bsz_t ttl_size {0}; - - std::unordered_map> new_dic; - - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - const auto& tgt_val = tgt_col[idx]; - - if (new_dic.find(src_val) == new_dic.end()) { - new_dic.insert({src_val, {}}); - } else { - new_dic[src_val].insert(tgt_val); - } - } - - sz_t max_sz {0}; - for (const auto& map : new_dic) { - auto map_size = map.second.size(); - if (max_sz < map.second.size()) { - max_sz = map_size; - } - - ttl_size += map.second.size() * (sizeof(PT) + sizeof(ofs_t)); - } - - auto bits_per_value = RANGE_BIT(max_sz); - auto ttl_bytes = (src_col.size() * bits_per_value / 8); - return ttl_size + ttl_bytes; -} - -template -bsz_t dict_fac::exp_multi_dic_create(const std::vector& src_col, const std::vector& tgt_col) { - FLS_ASSERT_EQUALITY(src_col.size(), tgt_col.size()) - - bsz_t ttl_size {0}; - - std::unordered_map> new_dic; - - for (n_t idx {0}; idx < src_col.size(); ++idx) { - const auto& src_val = src_col[idx]; - const auto& tgt_val = tgt_col[idx]; - - if (new_dic.find(src_val) == new_dic.end()) { - new_dic.insert({src_val, {}}); - } else { - new_dic[src_val].insert(tgt_val); - } - } - - sz_t max_sz {0}; - for (const auto& map : new_dic) { - auto map_size = map.second.size(); - if (max_sz < map.second.size()) { - max_sz = map_size; - } - - ttl_size += map.second.size() * (sizeof(PT) + sizeof(ofs_t)); - } - - auto bits_per_value = RANGE_BIT(max_sz); - auto ttl_bytes = (src_col.size() * bits_per_value / 8); - return ttl_size + ttl_bytes; -} - -/*--------------------------------------------------------------------------------------------------------------------*\ -+ Specialization -\*--------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(dict_fac) -} // namespace fastlanes diff --git a/src/cor/lyt/page/CMakeLists.txt b/src/cor/lyt/page/CMakeLists.txt deleted file mode 100644 index bbf75bbd..00000000 --- a/src/cor/lyt/page/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_library(page - OBJECT - page.cpp - page_hdr.cpp - page_param.cpp - entry_point.cpp - type.cpp - offset.cpp -) - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -if (ENABLE_IWYU) - set_property(TARGET page PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () diff --git a/src/cor/lyt/page/entry_point.cpp b/src/cor/lyt/page/entry_point.cpp deleted file mode 100644 index 66b0b635..00000000 --- a/src/cor/lyt/page/entry_point.cpp +++ /dev/null @@ -1,65 +0,0 @@ -#include "fls/cor/lyt/page/entry_point.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include -#include - -namespace fastlanes { -void EpArr::InitDecompress(Page& page) { - FLS_ASSERT_NOT_NULL_POINTER(page.start_p) - FLS_ASSERT_CORRECT_N(page.vec_c()) - FLS_ASSERT_CORRECT_N(page.arr_c) - - m_vec_c = page.vec_c(); - m_arr_c = page.arr_c; - m_vec_num = 0; - m_start_p = page.start_p + page.off_arr.ttl_sz() + CFG::PG::HDR_SZ; -} - -void EpArr::load_and_next() { - FLS_ASSERT_CORRECT_NUM(m_vec_num, m_vec_c) - FLS_ASSERT_NOT_NULL_POINTER(m_start_p) - FLS_ASSERT_CORRECT_N(m_vec_c) - - FLS_PLOG_KEY_VALUE("vec_n", std::to_string(m_vec_num)) - FLS_PLOG_KEY_VALUE("vec_c", std::to_string(m_vec_c)) - - /* the first ep_arr is always 0 and has not been stored! */ - if (m_vec_num == 0) { - memset(arr, 0, m_arr_c * CFG::PG::ENTRY_POINT_SZ); - m_vec_num++; - FLS_PLOG_KEY_VALUE("ep", ToStr::to_str(*this)) - return; - } - - memcpy(arr, m_start_p, m_arr_c * CFG::PG::ENTRY_POINT_SZ); - FLS_PLOG_KEY_VALUE("ep", ToStr::to_str(*this)) - m_start_p += m_arr_c * CFG::PG::ENTRY_POINT_SZ; - m_vec_num++; -} - -void EpArr::InitCompress(Page& page) { - FLS_ASSERT_CORRECT_N(page.arr_c) - - m_arr_c = page.arr_c; - m_vec_c = 0; - m_vec_num = 0; - - /* init all entry points to 0. */ - for (auto& ep : arr) { - ep = 0; - } -} - -n_t EpArr::sz() { - return m_arr_c * CFG::PG::ENTRY_POINT_SZ; -} -void EpArr::Reset() { - m_vec_num = 0; - m_vec_c = 0; -} - -EpArr::EpArr() = default; -} // namespace fastlanes diff --git a/src/cor/lyt/page/offset.cpp b/src/cor/lyt/page/offset.cpp deleted file mode 100644 index 6588712f..00000000 --- a/src/cor/lyt/page/offset.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "fls/cor/lyt/page/offset.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include -#include - -namespace fastlanes { -OffArr::OffArr() = default; - -void OffArr::InitDecompress(Page& page) { - FLS_ASSERT_CORRECT_N(page.arr_c) - FLS_ASSERT_NOT_NULL_POINTER(page.start_p) - - m_arr_c = page.arr_c; - memcpy(arr, page.start_p + CFG::PG::HDR_SZ, m_arr_c * CFG::PG::OFFSET_SZ); - FLS_PLOG_KEY_VALUE("off_arr", ToStr::to_str(*this).c_str()) -} - -void OffArr::InitCompress(n_t arr_c) { - FLS_ASSERT_CORRECT_N(arr_c) - m_arr_c = arr_c; -} - -n_t OffArr::ttl_sz() { - FLS_ASSERT_CORRECT_SZ(m_arr_c) - return m_arr_c * CFG::PG::OFFSET_SZ; -} -n_t OffArr::arr_c() { - return m_arr_c; -} -} // namespace fastlanes diff --git a/src/cor/lyt/page/page.cpp b/src/cor/lyt/page/page.cpp deleted file mode 100644 index f9a75875..00000000 --- a/src/cor/lyt/page/page.cpp +++ /dev/null @@ -1,181 +0,0 @@ -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/eng/hzl_cmp.hpp" -#include "fls/cor/exp/exp.hpp" -#include "fls/cor/exp/rpn.hpp" -#include "fls/cor/lyt/rec_hdr.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -Page::Page() - : m_ttl_sz {0} { -} - -void Page::Reset() { - for (auto& buf : buf_arr) { - buf.Reset(); - } - tup_c = 0; - arr_c = 0; - m_ttl_sz = 0; - pg_hdr.Reset(); - ep_arr.Reset(); -} - -void Page::InitCompress(const sp& exp_sp) { - for (auto& buff : buf_arr) { - FLS_UNUSED(buff.Rewind(0)); - } - ep_arr_buf.Rewind(0); - - /* Page hdr.*/ - pg_hdr = exp_sp->page_hdr(); - - /* Arr offsets: */ - arr_c = exp_sp->arw_c(); - off_arr.InitCompress(arr_c); - - /* Entry Point: */ - ep_arr.InitCompress(*this); - - /* Current size.*/ - m_ttl_sz = 0; - m_ttl_sz += CFG::PG::HDR_SZ; - m_ttl_sz += arr_c * CFG::PG::OFFSET_SZ; - - /* Add rec buff hdrs*/ - auto* rpn = exp_sp->cmpr_rpn(); - for (n_t i {0}; i < rpn->des_buf_c; ++i) { - if (!rpn->is_compressed[i]) { - continue; - } - - m_ttl_sz += sizeof(rec_hdr); - } - - /* Start pointer*/ - start_p = nullptr; - - FLS_PLOG_KEY_VALUE("initial_sz", std::to_string(m_ttl_sz)) -} - -void Page::InitDecompress(const uint8_t* p, const sp& exp_up) { - FLS_ASSERT_NOT_NULL_POINTER(p) - - /* Start pointer*/ - start_p = const_cast(p); - - /* Page hdr.*/ - pg_hdr.Load(p); - - /* Arr offsets: */ - arr_c = exp_up->arw_c(); - off_arr.InitDecompress(*this); - - /* Init Params: */ - params.Init(*this); - - /* Init EP: */ - ep_arr.InitDecompress(*this); - - tup_c = 0; - - FLS_PLOG_KEY_VALUE("pg", ToStr::to_str(*this)); -} - -PageHdr Page::hdr() { - return pg_hdr; -} - -void Page::Absorb(Vec& vec) { - if (vec_c() >= 1) { - ep_arr_buf.UnsafeAppend(ep_arr.arr, ep_arr.sz()); - m_ttl_sz += ep_arr.sz(); - } - - n_t vec_ttl_sz {0}; - for (size_t i {0}; i < arr_c; ++i) { - auto added_sz = vec.buf_arr[i].length(); - buf_arr[i].UnsafeAppend(vec.buf_arr[i].data(), added_sz); - ep_arr.arr[i] = ep_arr.arr[i] + added_sz; - vec_ttl_sz += added_sz; - } - - m_ttl_sz += vec_ttl_sz; - pg_hdr.vec_c++; - - FLS_PLOG_KEY_VALUE("ep", ToStr::to_str(ep_arr)) - FLS_PLOG_KEY_VALUE("ep_arr", - ToStr::to_str(reinterpret_cast(ep_arr_buf.mutable_data()), pg_hdr.vec_c)) - FLS_PLOG_KEY_VALUE("ttl_sz", std::to_string(m_ttl_sz)); - FLS_PLOG_KEY_VALUE("vec_ttl_sz", std::to_string(vec_ttl_sz)); - FLS_PLOG_KEY_VALUE("vec_c", std::to_string(vec_c())); - - vec.Reset(); -} - -void Page::vec_c_inc() { - pg_hdr.vec_c++; -} - -uint16_t Page::vec_c() { - return pg_hdr.vec_c; -} - -void Page::OffsetToSink(Buf& sink) { - /* First one - * Rest follow each other.*/ - off_arr.arr[0] = CFG::PG::HDR_SZ + off_arr.ttl_sz() + ep_arr_buf.length(); - for (size_t i {1}; i < arr_c; ++i) { - off_arr.arr[i] = off_arr.arr[i - 1] + buf_arr[i - 1].length(); - } - - sink.UnsafeAppend(off_arr.arr, off_arr.ttl_sz()); - FLS_PLOG_KEY_VALUE("OFF_arr", ToStr::to_str(off_arr)) -} - -PageParam Page::page_params() { - PageParam res; - ep_arr.load_and_next(); - for (size_t i {0}; i < arr_c; ++i) { - res.arr[i] = params.arr[i] + ep_arr.arr[i]; - } - - return res; -} - -n_t Page::size_of(Vec& vec) { - n_t vec_ttl_sz {0}; - - if (vec_c() >= 1) { - vec_ttl_sz = ep_arr.sz(); - } - - for (size_t i {0}; i < arr_c; ++i) { - auto added_sz = vec.buf_arr[i].length(); - vec_ttl_sz += added_sz; - } - - return vec_ttl_sz; -} - -n_t Page::get_total_sz() { - n_t ttl_sz {0}; - for (auto& buf : buf_arr) { - ttl_sz += buf.length(); - } - - return ttl_sz; -} - -bool Page::Verify() { - auto ttl_sz = get_total_sz(); - return ttl_sz <= CFG::PG::SZ; -} - -Page::~Page() = default; -} // namespace fastlanes diff --git a/src/cor/lyt/page/page_hdr.cpp b/src/cor/lyt/page/page_hdr.cpp deleted file mode 100644 index cf2a7a86..00000000 --- a/src/cor/lyt/page/page_hdr.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "fls/cor/lyt/page/page_hdr.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/common/assert.hpp" -#include - -namespace fastlanes { -void PageHdr::Load(const uint8_t* p) { - FLS_ASSERT_NOT_NULL_POINTER(p) - - *this = *reinterpret_cast(p); -} - -uint32_t PageHdr::Serialize() { - return 0; -} - -void PageHdr::AddCompressedArrBitMap() { -} -void PageHdr::Reset() { - vec_c = 0; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Static Assert: -\---------------------------------------------------------------------------------------------------------------------*/ -static_assert(std::is_standard_layout() && std::is_trivial(), - "PageHdr should be a standard-layout trivial type."); -static_assert(sizeof(PageHdr) == CFG::PG::HDR_SZ, "PageHdr should be 4-byte long."); -} // namespace fastlanes diff --git a/src/cor/lyt/page/page_param.cpp b/src/cor/lyt/page/page_param.cpp deleted file mode 100644 index 0990c483..00000000 --- a/src/cor/lyt/page/page_param.cpp +++ /dev/null @@ -1,13 +0,0 @@ -#include "fls/cor/lyt/page/page_param.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include - -namespace fastlanes { -void PageParam::Init(Page& page) { - /* Pointer = start_p + offset! */ - for (size_t i {0}; i < page.arr_c; ++i) { - arr[i] = page.start_p + page.off_arr.arr[i]; - } -} -} // namespace fastlanes diff --git a/src/cor/lyt/page/type.cpp b/src/cor/lyt/page/type.cpp deleted file mode 100644 index cd8f4deb..00000000 --- a/src/cor/lyt/page/type.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "fls/cor/lyt/page/type.hpp" -#include "fls/common/common.hpp" -#include - -namespace fastlanes { -std::string to_string(PageT t) { - switch (t) { - case PageT::COMPRESS: - return {"COMPRESS"}; - case PageT::DECOMPRESS: - return {"DECOMPRESS"}; - case PageT::INVALID: - default: - return {"INVALID"}; - } - - return {}; -} -} // namespace fastlanes diff --git a/src/cor/lyt/vec.cpp b/src/cor/lyt/vec.cpp deleted file mode 100644 index 9c9ca594..00000000 --- a/src/cor/lyt/vec.cpp +++ /dev/null @@ -1,365 +0,0 @@ -#include "fls/cor/lyt/vec.hpp" -#include "fls/common/string.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/eng/engine.hpp" -#include "fls/cor/lyt/dic/dic.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/prm/fsst/fsst_prm.hpp" -#include "fls/cor/prm/fsst12/fsst12_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include - -#pragma clang diagnostic ignored "-Wshorten-64-to-32" - -namespace fastlanes { -static constexpr uint64_t VEC_SZ = 1024ULL * 1024; - -Vec::Vec() - : buf_arr {Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ), Buf(VEC_SZ)} - , arr_c {0} - , tup_c {0} { -} - -void Vec::Reset() { - for (auto& buff : buf_arr) { - buff.Rewind(0); - } - - arr_c = 0; -} - -void Vec::FlushToPage(Page& page) { - for (size_t i {0}; i < arr_c; ++i) { - n_t sz = buf_arr[i].length(); - page.buf_arr[i].UnsafeAppend(buf_arr[i].data(), buf_arr[i].length()); - page.ep_arr_buf.UnsafeAppend(&sz, sizeof(uint16_t)); - } -} - -template -void Vec::Init(const T* in_arr, EngineState& stt) { - if constexpr (std::is_arithmetic_v) { - FLS_ASSERT_NOT_NULL_POINTER(in_arr) - - auto exp_t = stt.analyze_state_up->m_exp_t; - FLS_ASSERT_CORRECT_EXP_T(static_cast(exp_t)) - - if constexpr (std::is_same_v || std::is_same_v) { - Reset(); - switch (exp_t) { - case ExpT::RLE: { - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - auto cur_val = in_arr[i]; - - if (i == 0) { - cur_val = in_arr[0]; - pos_val = 0; - buf_arr[0].UnsafeAppend(&cur_val, sizeof(T)); - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - continue; - } - - if (cur_val != in_arr[i - 1]) { - buf_arr[0].UnsafeAppend(&cur_val, sizeof(T)); - pos_val += 1; - } - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - } - - return; - } - case ExpT::DICT_RLE: { - auto idx_vec = - stt.analyze_state_up->dic_up->get_idx_vec(nullptr, reinterpret_cast(in_arr)); - - auto* idx_arr = reinterpret_cast(idx_vec->data()); - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - int64_t cur_val = idx_arr[i]; - - if (i == 0) { - cur_val = idx_arr[0]; - pos_val = 0; - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - continue; - } - - if (cur_val != idx_arr[i - 1]) { - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - pos_val += 1; - } - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - } - arr_c += 1; - } - case ExpT::DICT_VAL: - case ExpT::DICT_FRQ: - case ExpT::DICT: { - auto idx_arr = stt.analyze_state_up->dic_up->get_idx_vec(nullptr, in_arr); - - buf_arr[arr_c + 0].UnsafeAppend(idx_arr->data(), vec_bsz()); - - arr_c += 1; - return; - } - case ExpT::UNCOMPRESSED: - case ExpT::BYTE_ARR: - case ExpT::FFOR: - case ExpT::ALP: - case ExpT::DELTA: { - [[maybe_unused]] auto* val_arr = reinterpret_cast(buf_arr[0].mutable_data()); - buf_arr[arr_c + 0].UnsafeAppend(in_arr, vec_bsz()); - - arr_c += 1; - FLS_PLOG_KEY_VALUE("val_arr", ToStr::to_str(val_arr)); - return; - } - case ExpT::TGT_EQUALITY: - case ExpT::TGT_1T1: - case ExpT::TGT_1TN: - case ExpT::SRC_1T1: - case ExpT::SRC_1TN: - case ExpT::ALP_RD: - break; - case ExpT::INVALID: - case ExpT::UNDECIDED: - case ExpT::FSST: - default: { - FLS_ABORT("EXP IS NOT SUPPORTED!") - return; - } - } - } - } - /**/ - else if (std::is_same_v) { - this->Reset(); - - auto exp_t = stt.analyze_state_up->m_exp_t; - FLS_ASSERT_CORRECT_EXP_T(static_cast(exp_t)) - - /*two buffs: hacky */ - Buf bytes; - Buf smart_offsets; - - /*Extract bytes*/ - for (idx_t idx {0}; idx < vec_n_tup(); ++idx) { - auto str = in_arr[idx]; - bytes.UnsafeAppend(str.data(), str.size()); - } - - /* Extract Offsets. - * Smart Offsets - */ - ofs_t cur_ofs {0}; - for (auto i {0ULL}; i < vec_n_tup(); ++i) { - cur_ofs += static_cast(in_arr[i].size()); - smart_offsets.UnsafeAppend(&cur_ofs, sizeof(ofs_t)); - } - - switch (exp_t) { - case ExpT::FSST: { - auto* encoder = reinterpret_cast(stt.analyze_state_up->fsst_encoder); - auto* str_arr = bytes.mutable_data(); - auto* off_arr = smart_offsets.mutable_data(); - auto str_p_in = Str::smart_offset_to_pointer(vec_sz(), off_arr, str_arr); - auto str_len_in = fsst_wrapper::offset_to_length(vec_sz(), off_arr); - - auto* out_p = buf_arr[arr_c + 0].mutable_data(); - auto* len_out = reinterpret_cast(buf_arr[arr_c + 3].mutable_data()); - auto* off_out = reinterpret_cast(buf_arr[arr_c + 2].mutable_data()); - /* - fsst_encoder_t *encoder, IN: encoder obtained from fsst_create(). - size_t nstrings, IN: number of strings in batch to compress. - size_t len_in[], IN: byte-lengths of the inputs - unsigned char *str_in[], IN: input string start pointers. - size_t outsize, IN: byte-length of output buffer. - unsigned char *output, OUT: memory buffer to put the compressed strings in (one after the other). - size_t len_out[], OUT: byte-lengths of the compressed strings. - unsigned char *str_out[] OUT: output string start pointers. Will all point into [output,output+size) - .*/ - - fsst_compress(encoder, // - vec_n_tup(), - str_len_in->data(), - str_p_in->data(), - smart_offsets.Capacity(), - out_p, - len_out, - off_out); - - // smart-offsets. - for (n_t i = 0; i < vec_n_tup() - 1; ++i) { - len_out[i] = static_cast(off_out[i + 1] - out_p); - FLS_ASSERT_CORRECT_SMART_OFFSET(off_out[i + 1] - out_p) - } - len_out[1023] = len_out[1022] + len_out[1023]; - - buf_arr[arr_c + 1].UnsafeAppend(len_out, vec_bsz()); - - arr_c += 2; - - FLS_PLOG_KEY_VALUE("byte_arr", ToStr::to_str(buf_arr[0].mutable_data())); - FLS_PLOG_KEY_VALUE("len_arr", ToStr::to_str(buf_arr[1].mutable_data())); - FLS_PLOG_KEY_VALUE("str_pointer", ToStr::to_str(buf_arr[2].mutable_data())); - } break; - case ExpT::FSST12: { - auto* encoder = reinterpret_cast(stt.analyze_state_up->fsst12_encoder); - auto* str_arr = bytes.mutable_data(); - auto* off_arr = smart_offsets.mutable_data(); - auto str_p_in = Str::smart_offset_to_pointer(vec_sz(), off_arr, str_arr); - auto str_len_in = fsst_wrapper::offset_to_length(vec_sz(), off_arr); - - auto* out_p = buf_arr[arr_c + 0].mutable_data(); - auto* len_out = reinterpret_cast(buf_arr[arr_c + 3].mutable_data()); - auto* off_out = reinterpret_cast(buf_arr[arr_c + 2].mutable_data()); - - /* Compress a batch of strings (on AVX512 machines best performance is obtained by compressing more than - * 32KB of string volume). */ - /* The output buffer must be large; at least "conservative space" (7+2*inputlength) for the first string for - * something to happen. */ - /* OUT: the number of compressed strings (<=n) that fit the output buffer. */ - /* IN: encoder obtained from fsst12_create(). */ - /* IN: number of strings in batch to compress. */ - /* IN: byte-lengths of the inputs */ - /* IN: input string start pointers. */ - /* IN: byte-length of output buffer. */ - /* OUT: memory buffer to put the compressed strings in (one after the other). */ - /* OUT: byte-lengths of the compressed strings. */ - /* OUT: output string start pointers. Will all point into [output,output+size). */ - - fsst12_compress(encoder, // - vec_n_tup(), - str_len_in->data(), - str_p_in->data(), - smart_offsets.Capacity(), - out_p, - len_out, - off_out); - - // smart-offsets. - for (n_t i = 0; i < vec_n_tup() - 1; ++i) { - len_out[i] = static_cast(off_out[i + 1] - out_p); - FLS_ASSERT_CORRECT_SMART_OFFSET(off_out[i + 1] - out_p) - } - len_out[1023] = len_out[1022] + len_out[1023]; - - buf_arr[arr_c + 1].UnsafeAppend(len_out, vec_bsz()); - - arr_c += 2; - - FLS_PLOG_KEY_VALUE("byte_arr", ToStr::to_str(buf_arr[0].mutable_data())); - FLS_PLOG_KEY_VALUE("len_arr", ToStr::to_str(buf_arr[1].mutable_data())); - FLS_PLOG_KEY_VALUE("str_pointer", ToStr::to_str(buf_arr[2].mutable_data())); - } break; - case ExpT::DICT_VAL: - case ExpT::DICT_FRQ: - case ExpT::DICT: { - auto idx_arr = stt.analyze_state_up->dic_up->get_idx_vec(smart_offsets.data(), bytes.data()); - - buf_arr[arr_c + 0].UnsafeAppend(idx_arr->data(), vec_bsz()); - - arr_c += 1; - - FLS_PLOG_KEY_VALUE("idx_arr", - ToStr::to_str(reinterpret_cast(buf_arr[0].mutable_data()))); - } break; - case ExpT::DICT_RLE: { - auto idx_vec = stt.analyze_state_up->dic_up->get_idx_vec(smart_offsets.data(), bytes.data()); - - auto* idx_arr = reinterpret_cast(idx_vec->data()); - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - int64_t cur_val = idx_arr[i]; - - if (i == 0) { - cur_val = idx_arr[0]; - pos_val = 0; - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - continue; - } - - if (cur_val != idx_arr[i - 1]) { - buf_arr[0].UnsafeAppend(&cur_val, sizeof(idx_t)); - pos_val += 1; - } - buf_arr[1].UnsafeAppend(&pos_val, sizeof(rle_idx_t)); - } - arr_c += 2; - } break; - case ExpT::BYTE_ARR: { - buf_arr[arr_c + 0].UnsafeAppend(bytes.data(), bytes.length()); - buf_arr[arr_c + 1].Clone(smart_offsets); - arr_c += 2; - - FLS_PLOG_KEY_VALUE("byte_arr", ToStr::to_str(buf_arr[0].mutable_data())); - FLS_PLOG_KEY_VALUE("offset_arr", ToStr::to_str(buf_arr[1].mutable_data())); - } break; - default: - FLS_ABORT("EXP IS NOT SUPPORTED!") - } - } - /**/ - else { - FLS_ABORT("NOT SUPPORTED TYPE") - } -} - -void Vec::Log() { - // FLS_PRINT_4MSG("tup_c: ", std::to_string(tup_c), "arr_c", std::to_string(arr_c)) -} - -VecParam Vec::vec_param() { - return {buf_arr[0].mutable_data(), buf_arr[1].mutable_data(), buf_arr[2].mutable_data()}; -} - -template -void Vec::flatten_to(T* out_arr) { - FLS_ASSERT_NOT_NULL_POINTER(out_arr) - FLS_ASSERT_NOT_NULL_POINTER(buf_arr[0].data()) - FLS_ASSERT_NOT_NULL_POINTER(buf_arr[1].data()) - - auto* val_arr = reinterpret_cast(buf_arr[0].mutable_data()); - auto* pos_arr = reinterpret_cast(buf_arr[1].mutable_data()); - - for (n_t i {0}; i < vec_n_tup(); ++i) { - out_arr[i] = val_arr[pos_arr[i]]; - } - - FLS_PLOG_KEY_VALUE("val_arr", ToStr::to_str(val_arr)); - FLS_PLOG_KEY_VALUE("pos_arr", ToStr::to_str(pos_arr)); -} - -Vec::~Vec() = default; - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -// clang-format off -template void Vec::Init(const uint8_t *p,EngineState& stt); -template void Vec::Init(const uint16_t *p,EngineState& stt); -template void Vec::Init(const uint32_t *p,EngineState& stt); -template void Vec::Init(const uint64_t *p,EngineState& stt); -template void Vec::Init(const int8_t *p,EngineState& stt); -template void Vec::Init(const int16_t *p,EngineState& stt); -template void Vec::Init(const int32_t *p,EngineState& stt); -template void Vec::Init(const int64_t *p,EngineState& stt); -template void Vec::Init(const float *p,EngineState& stt); -template void Vec::Init(const double *p,EngineState& stt); -template void Vec::Init(const bool *p,EngineState& stt); -template void Vec::Init(const str_pt *p,EngineState& stt); -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -// clang-format off -template void Vec::flatten_to(dbl_pt *out_p); -template void Vec::flatten_to(uint64_t *out_p); -template void Vec::flatten_to(uint32_t *out_p); -template void Vec::flatten_to(uint16_t *out_p); -template void Vec::flatten_to(uint8_t *out_p); - -} // namespace fastlanes diff --git a/src/cor/mtd.cpp b/src/cor/mtd.cpp deleted file mode 100644 index e205f799..00000000 --- a/src/cor/mtd.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "fls/cor/mtd.hpp" -#include "fls/cfg/cfg.hpp" - -namespace fastlanes { -template -Mtd::Mtd(n_t cap) - : cap {cap} - , size {0} - , compressed_sz {0} - , min_max {} - , histogram {} { -} - -template -n_t Mtd::Count() const { - return size.Count(); -} - -template -double Mtd::Ratio() const { - return size / compressed_sz; -} - -template -n_t Mtd::UnusedSize() const { - return size - compressed_sz; -} - -template -T Mtd::Min() const { - return min_max.min; -} - -template -T Mtd::Max() const { - return min_max.max; -} - -template -void Mtd::Reset() { - min_max.Reset(); - histogram.Reset(); - compressed_sz.reset(); -} - -template -n_t Mtd::VecCount() const { - return Count() / fastlanes::CFG::VEC_TUP_C; -} - -template -double Mtd::Occupancy() const { - return compressed_sz / cap; -} - -template -n_t Mtd::Cardinality() const { - FLS_ASSERT(histogram.val_vec.size() != 0, "Hist.size is zero. Call cal before use!", " "); - - return histogram.val_vec.size(); -} - -template -n_t Mtd::Size() const { - return size.Val(); -} -template -void Mtd::Reset(n_t c) { - min_max.Reset(); - histogram.Reset(); - compressed_sz.reset(); - size = byte_c {c * sizeof(T)}; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(Mtd) -// CHECK -// { -// template class Mtd; -// template class Mtd; -// template class Mtd; -// } -} // namespace fastlanes diff --git a/src/cor/prm/CMakeLists.txt b/src/cor/prm/CMakeLists.txt index d5ff4f03..17761234 100644 --- a/src/cor/prm/CMakeLists.txt +++ b/src/cor/prm/CMakeLists.txt @@ -3,8 +3,7 @@ add_subdirectory(fsst12) add_library(fls_all_prms OBJECT - prm.cpp - type.cpp) + prm.cpp) set(FASTLANES_OBJECT_FILES ${FASTLANES_OBJECT_FILES} $ diff --git a/src/cor/prm/alp/CMakeLists.txt b/src/cor/prm/alp/CMakeLists.txt deleted file mode 100644 index 85deac7e..00000000 --- a/src/cor/prm/alp/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -add_library(fls_alp_prm - OBJECT - alp_decompress.cpp - alp_compress.cpp) - -if (ENABLE_IWYU) - set_target_properties(fls_alp_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_alp_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -target_link_libraries(fls_alp_prm - PUBLIC - FastLanes::headers -) diff --git a/src/cor/prm/alp/alp_compress.cpp b/src/cor/prm/alp/alp_compress.cpp deleted file mode 100644 index ca67d822..00000000 --- a/src/cor/prm/alp/alp_compress.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "alp.hpp" -#include "fls/cor/eng/analyzer.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/engine.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/alp_prm.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/primitive/ffor/ffor.hpp" - -namespace fastlanes { -static void alp_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - - // auto* dbl_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - auto& alp_stt = *stt.eng_state.analyze_state_up->alp_dbl_state_up; - - // auto* exc_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 0].mutable_data()); - // auto* pos_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 1].mutable_data()); - auto* exc_c_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 2].mutable_data()); - auto* base_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 3].mutable_data()); - auto* ffor_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 5].mutable_data()); - - uint8_t bw = 0; - int64_t tmp_dig_arr[vec_sz()]; // TODO BUFFER_POOl - // alp::encoder::encode(dbl_arr, exc_arr, pos_arr, exc_c_arr, tmp_dig_arr, alp_stt); TODO - alp::encoder::analyze_ffor(tmp_dig_arr, bw, base_arr); - generated::ffor::fallback::scalar::ffor(tmp_dig_arr, ffor_arr, bw, base_arr); - - auto exc_c = exc_c_arr[0]; - alp_mtd_t alp_mtd {bw, alp_stt.exp, alp_stt.fac}; - - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAdvance(exc_c * sizeof(uint16_t)); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAdvance(exc_c * sizeof(uint16_t)); - des_vec.buf_arr[stt.cur_des_buff + 2].UnsafeAdvance(sizeof(uint16_t)); - des_vec.buf_arr[stt.cur_des_buff + 3].UnsafeAdvance(sizeof(int64_t)); - des_vec.buf_arr[stt.cur_des_buff + 4].UnsafeAppend(&alp_mtd, sizeof(alp_mtd_t)); - des_vec.buf_arr[stt.cur_des_buff + 5].UnsafeAdvance(ffor_prm::vec_sz(bw)); - - stt.cur_des_buff += 6; -} - -template -cmpr_fun_t alp_prm::ResolveCompressFunc() { - if constexpr (std::is_same_v) { - return alp_compress; - } else { - FLS_ABORT("ONLY WORKS FOR DOUBLE") - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, alp_prm::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/alp/alp_decompress.cpp b/src/cor/prm/alp/alp_decompress.cpp deleted file mode 100644 index 58a1621c..00000000 --- a/src/cor/prm/alp/alp_decompress.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "alp.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/alp_prm.hpp" -#include "fls/primitive/unffor/unffor.hpp" - -namespace fastlanes { -static void alp_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 5]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 4]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 3]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 2]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 1]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 0]) - - auto* ffor_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* alp_mtd_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* base_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 2]); - // auto* exc_c_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 3]); - // auto* pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 4]); - // auto* exc_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 5]); - - auto* dec_dbl_arr = reinterpret_cast(des.arr[stt.cur_des_arr]); - - int64_t unffor_arr[vec_sz()]; // TODO BUFFER_POOl - auto alp_mtd = alp_mtd_arr[0]; - - generated::unffor::fallback::scalar::unffor(ffor_arr, unffor_arr, alp_mtd.bw, base_arr); - alp::decoder::decode(unffor_arr, alp_mtd.fac, alp_mtd.exp, dec_dbl_arr); - // alp::decoder::patch_exceptions(dec_dbl_arr, exc_arr, pos_arr, exc_c_arr); TODO - - stt.cur_src_arr -= 6; -} - -template -de_cmpr_fun_t alp_prm::ResolveDecompressFunc() { - if constexpr (std::is_same_v) { - return alp_decompress; - } else { - FLS_ABORT("ONLY WORKS FOR DOUBLE") - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, alp_prm::ResolveDecompressFunc) -} // namespace fastlanes diff --git a/src/cor/prm/b_ffor/CMakeLists.txt b/src/cor/prm/b_ffor/CMakeLists.txt deleted file mode 100644 index 7c91d916..00000000 --- a/src/cor/prm/b_ffor/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -add_library(fls_b_ffor_prm - OBJECT - b_ffor_decompress.cpp - b_ffor_compress.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_b_ffor_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_b_ffor_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -target_link_libraries(fls_b_ffor_prm - PUBLIC - FastLanes::headers -) diff --git a/src/cor/prm/b_ffor/b_ffor_compress.cpp b/src/cor/prm/b_ffor/b_ffor_compress.cpp deleted file mode 100644 index 52d12ed3..00000000 --- a/src/cor/prm/b_ffor/b_ffor_compress.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/b_ffor_prm.hpp" -#include "fls/ffor.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void ffor_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { -} - -template -cmpr_fun_t b_ffor_prm::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, b_ffor_prm::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/b_ffor/b_ffor_decompress.cpp b/src/cor/prm/b_ffor/b_ffor_decompress.cpp deleted file mode 100644 index e60b2315..00000000 --- a/src/cor/prm/b_ffor/b_ffor_decompress.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/b_ffor_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/unffor/unffor.hpp" - -namespace fastlanes { -template -static void b_ffor_decompress(PageParam src, VecParam des, DecompressState& stt) { -} - -template -de_cmpr_fun_t b_ffor_prm::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } else if constexpr (std::is_same()) { - return b_ffor_decompress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc) -// FLS_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc, parquet::FixedLenByteArray) -// FLS_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc, parquet::Int96) -// FLS_FTS(de_cmpr_fun_t, b_ffor_prm::ResolveDecompressFunc, parquet::ByteArray) -} // namespace fastlanes diff --git a/src/cor/prm/bitpack/CMakeLists.txt b/src/cor/prm/bitpack/CMakeLists.txt deleted file mode 100644 index 87f9358c..00000000 --- a/src/cor/prm/bitpack/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -add_library(fls_bitpack - OBJECT - bitpack_decompress.cpp - bitpack_compress.cpp) - -target_compile_definitions(fls_bitpack PRIVATE IS_SCALAR) -target_compile_options(fls_bitpack PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_bitpack PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_bitpack PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) - -target_link_libraries(fls_bitpack - PUBLIC - FastLanes::headers -) diff --git a/src/cor/prm/bitpack/bitpack_compress.cpp b/src/cor/prm/bitpack/bitpack_compress.cpp deleted file mode 100644 index 19cd74a0..00000000 --- a/src/cor/prm/bitpack/bitpack_compress.cpp +++ /dev/null @@ -1,22254 +0,0 @@ -#include "fls/cor/prm/bitpack.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t -#include // for __restrict -namespace fastlanes { namespace bitpack { -void pack_1bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 1U; - src = *(in + 128 * 2 + i); - tmp |= src << 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 3U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 5U; - src = *(in + 128 * 6 + i); - tmp |= src << 6U; - src = *(in + 128 * 7 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 2U; - src = *(in + 128 * 2 + i); - tmp |= src << 4U; - src = *(in + 128 * 3 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - src = *(in + 128 * 5 + i); - tmp |= src << 2U; - src = *(in + 128 * 6 + i); - tmp |= src << 4U; - src = *(in + 128 * 7 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_3bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 3U; - src = *(in + 128 * 2 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src >> 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 1U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src >> 1U; - src = *(in + 128 * 6 + i); - tmp |= src << 2U; - src = *(in + 128 * 7 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_4bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src; - src = *(in + 128 * 3 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - src = *(in + 128 * 5 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src; - src = *(in + 128 * 7 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_5bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src >> 3U; - src = *(in + 128 * 2 + i); - tmp |= src << 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 3 + i); - tmp = src >> 1U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src >> 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 1U; - src = *(in + 128 * 6 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src >> 2U; - src = *(in + 128 * 7 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_6bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src >> 2U; - src = *(in + 128 * 2 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src >> 4U; - src = *(in + 128 * 3 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - src = *(in + 128 * 5 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src >> 2U; - src = *(in + 128 * 6 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src >> 4U; - src = *(in + 128 * 7 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_7bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - src = *(in + 128 * 1 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src >> 1U; - src = *(in + 128 * 2 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src >> 2U; - src = *(in + 128 * 3 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 3 + i); - tmp = src >> 3U; - src = *(in + 128 * 4 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src >> 4U; - src = *(in + 128 * 5 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src >> 5U; - src = *(in + 128 * 6 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src >> 6U; - src = *(in + 128 * 7 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_8bit_8simdWidth(uint8_t* __restrict in, uint8_t* __restrict out) { - uint8_t tmp = 0U; - uint8_t src; - for (int i = 0; i < 128; i++) { - src = *(in + 128 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 128; - src = *(in + 128 * 7 + i); - tmp = src; - *(out + i) = tmp; - out -= 896; - } -} -void pack_1bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 1U; - src = *(in + 64 * 2 + i); - tmp |= src << 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 3U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 5U; - src = *(in + 64 * 6 + i); - tmp |= src << 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 7U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 9U; - src = *(in + 64 * 10 + i); - tmp |= src << 10U; - src = *(in + 64 * 11 + i); - tmp |= src << 11U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - src = *(in + 64 * 13 + i); - tmp |= src << 13U; - src = *(in + 64 * 14 + i); - tmp |= src << 14U; - src = *(in + 64 * 15 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 2U; - src = *(in + 64 * 2 + i); - tmp |= src << 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 6U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 10U; - src = *(in + 64 * 6 + i); - tmp |= src << 12U; - src = *(in + 64 * 7 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 2U; - src = *(in + 64 * 10 + i); - tmp |= src << 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 6U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 10U; - src = *(in + 64 * 14 + i); - tmp |= src << 12U; - src = *(in + 64 * 15 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out -= 64; - } -} -void pack_3bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 3U; - src = *(in + 64 * 2 + i); - tmp |= src << 6U; - src = *(in + 64 * 3 + i); - tmp |= src << 9U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - src = *(in + 64 * 5 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 1U; - src = *(in + 64 * 6 + i); - tmp |= src << 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 5U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 11U; - src = *(in + 64 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 1U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 7U; - src = *(in + 64 * 14 + i); - tmp |= src << 10U; - src = *(in + 64 * 15 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_4bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 4U; - src = *(in + 64 * 2 + i); - tmp |= src << 8U; - src = *(in + 64 * 3 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - src = *(in + 64 * 5 + i); - tmp |= src << 4U; - src = *(in + 64 * 6 + i); - tmp |= src << 8U; - src = *(in + 64 * 7 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 4U; - src = *(in + 64 * 10 + i); - tmp |= src << 8U; - src = *(in + 64 * 11 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - src = *(in + 64 * 13 + i); - tmp |= src << 4U; - src = *(in + 64 * 14 + i); - tmp |= src << 8U; - src = *(in + 64 * 15 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out -= 192; - } -} -void pack_5bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 5U; - src = *(in + 64 * 2 + i); - tmp |= src << 10U; - src = *(in + 64 * 3 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 1U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 9U; - src = *(in + 64 * 6 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 3U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 3U; - src = *(in + 64 * 10 + i); - tmp |= src << 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 7U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 1U; - src = *(in + 64 * 14 + i); - tmp |= src << 6U; - src = *(in + 64 * 15 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_6bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 6U; - src = *(in + 64 * 2 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 2U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 2U; - src = *(in + 64 * 6 + i); - tmp |= src << 4U; - src = *(in + 64 * 7 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 6U; - src = *(in + 64 * 10 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 2U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 2U; - src = *(in + 64 * 14 + i); - tmp |= src << 4U; - src = *(in + 64 * 15 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out -= 320; - } -} -void pack_7bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 7U; - src = *(in + 64 * 2 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 5U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 3U; - src = *(in + 64 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 1U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 1U; - src = *(in + 64 * 10 + i); - tmp |= src << 6U; - src = *(in + 64 * 11 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 3U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 5U; - src = *(in + 64 * 14 + i); - tmp |= src << 2U; - src = *(in + 64 * 15 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_8bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src; - src = *(in + 64 * 3 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - src = *(in + 64 * 5 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src; - src = *(in + 64 * 7 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src; - src = *(in + 64 * 11 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - src = *(in + 64 * 13 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src; - src = *(in + 64 * 15 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out -= 448; - } -} -void pack_9bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 7U; - src = *(in + 64 * 2 + i); - tmp |= src << 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 5U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 3U; - src = *(in + 64 * 6 + i); - tmp |= src << 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 1U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 1U; - src = *(in + 64 * 10 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 6U; - src = *(in + 64 * 11 + i); - tmp |= src << 3U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 5U; - src = *(in + 64 * 14 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 2U; - src = *(in + 64 * 15 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_10bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 6U; - src = *(in + 64 * 2 + i); - tmp |= src << 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 2U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 2U; - src = *(in + 64 * 6 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 4U; - src = *(in + 64 * 7 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 6U; - src = *(in + 64 * 10 + i); - tmp |= src << 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 2U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 2U; - src = *(in + 64 * 14 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 4U; - src = *(in + 64 * 15 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 576; - } -} -void pack_11bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 5U; - src = *(in + 64 * 2 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 10U; - src = *(in + 64 * 3 + i); - tmp |= src << 1U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 9U; - src = *(in + 64 * 6 + i); - tmp |= src << 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 3U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 3U; - src = *(in + 64 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 7U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 1U; - src = *(in + 64 * 14 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 6U; - src = *(in + 64 * 15 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_12bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 4U; - src = *(in + 64 * 2 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 8U; - src = *(in + 64 * 3 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - src = *(in + 64 * 5 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 4U; - src = *(in + 64 * 6 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 8U; - src = *(in + 64 * 7 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 4U; - src = *(in + 64 * 10 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 8U; - src = *(in + 64 * 11 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - src = *(in + 64 * 13 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 4U; - src = *(in + 64 * 14 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 8U; - src = *(in + 64 * 15 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 704; - } -} -void pack_13bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 3U; - src = *(in + 64 * 2 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 6U; - src = *(in + 64 * 3 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 9U; - src = *(in + 64 * 4 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 12U; - src = *(in + 64 * 5 + i); - tmp |= src << 1U; - src = *(in + 64 * 6 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 2U; - src = *(in + 64 * 7 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 5U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 11U; - src = *(in + 64 * 10 + i); - tmp |= src << 2U; - src = *(in + 64 * 11 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 1U; - src = *(in + 64 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 4U; - src = *(in + 64 * 13 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 7U; - src = *(in + 64 * 14 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 10U; - src = *(in + 64 * 15 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_14bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 2U; - src = *(in + 64 * 2 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 4U; - src = *(in + 64 * 3 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 6U; - src = *(in + 64 * 4 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 8U; - src = *(in + 64 * 5 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 10U; - src = *(in + 64 * 6 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 12U; - src = *(in + 64 * 7 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - src = *(in + 64 * 9 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 2U; - src = *(in + 64 * 10 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 4U; - src = *(in + 64 * 11 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 6U; - src = *(in + 64 * 12 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 8U; - src = *(in + 64 * 13 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 10U; - src = *(in + 64 * 14 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 12U; - src = *(in + 64 * 15 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 832; - } -} -void pack_15bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - src = *(in + 64 * 1 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src >> 1U; - src = *(in + 64 * 2 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src >> 2U; - src = *(in + 64 * 3 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src >> 3U; - src = *(in + 64 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src >> 4U; - src = *(in + 64 * 5 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src >> 5U; - src = *(in + 64 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src >> 6U; - src = *(in + 64 * 7 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src >> 7U; - src = *(in + 64 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src >> 8U; - src = *(in + 64 * 9 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src >> 9U; - src = *(in + 64 * 10 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src >> 10U; - src = *(in + 64 * 11 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src >> 11U; - src = *(in + 64 * 12 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src >> 12U; - src = *(in + 64 * 13 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src >> 13U; - src = *(in + 64 * 14 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src >> 14U; - src = *(in + 64 * 15 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 896; - } -} -void pack_16bit_16simdWidth(uint16_t* __restrict in, uint16_t* __restrict out) { - uint16_t tmp = 0U; - uint16_t src; - for (int i = 0; i < 64; i++) { - src = *(in + 64 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 7 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 8 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 9 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 10 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 11 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 12 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 13 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 14 + i); - tmp = src; - *(out + i) = tmp; - out += 64; - src = *(in + 64 * 15 + i); - tmp = src; - *(out + i) = tmp; - out -= 960; - } -} -void pack_1bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 1U; - src = *(in + 32 * 2 + i); - tmp |= src << 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 3U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 5U; - src = *(in + 32 * 6 + i); - tmp |= src << 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 7U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 9U; - src = *(in + 32 * 10 + i); - tmp |= src << 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 11U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 13U; - src = *(in + 32 * 14 + i); - tmp |= src << 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 15U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 17U; - src = *(in + 32 * 18 + i); - tmp |= src << 18U; - src = *(in + 32 * 19 + i); - tmp |= src << 19U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 21U; - src = *(in + 32 * 22 + i); - tmp |= src << 22U; - src = *(in + 32 * 23 + i); - tmp |= src << 23U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 25U; - src = *(in + 32 * 26 + i); - tmp |= src << 26U; - src = *(in + 32 * 27 + i); - tmp |= src << 27U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - src = *(in + 32 * 29 + i); - tmp |= src << 29U; - src = *(in + 32 * 30 + i); - tmp |= src << 30U; - src = *(in + 32 * 31 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 2U; - src = *(in + 32 * 2 + i); - tmp |= src << 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 6U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 10U; - src = *(in + 32 * 6 + i); - tmp |= src << 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 14U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 18U; - src = *(in + 32 * 10 + i); - tmp |= src << 20U; - src = *(in + 32 * 11 + i); - tmp |= src << 22U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - src = *(in + 32 * 13 + i); - tmp |= src << 26U; - src = *(in + 32 * 14 + i); - tmp |= src << 28U; - src = *(in + 32 * 15 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 2U; - src = *(in + 32 * 18 + i); - tmp |= src << 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 6U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 10U; - src = *(in + 32 * 22 + i); - tmp |= src << 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 14U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 18U; - src = *(in + 32 * 26 + i); - tmp |= src << 20U; - src = *(in + 32 * 27 + i); - tmp |= src << 22U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - src = *(in + 32 * 29 + i); - tmp |= src << 26U; - src = *(in + 32 * 30 + i); - tmp |= src << 28U; - src = *(in + 32 * 31 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out -= 32; - } -} -void pack_3bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 3U; - src = *(in + 32 * 2 + i); - tmp |= src << 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 9U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 15U; - src = *(in + 32 * 6 + i); - tmp |= src << 18U; - src = *(in + 32 * 7 + i); - tmp |= src << 21U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 27U; - src = *(in + 32 * 10 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 1U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 7U; - src = *(in + 32 * 14 + i); - tmp |= src << 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 13U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 19U; - src = *(in + 32 * 18 + i); - tmp |= src << 22U; - src = *(in + 32 * 19 + i); - tmp |= src << 25U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - src = *(in + 32 * 21 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 1U; - src = *(in + 32 * 22 + i); - tmp |= src << 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 5U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 11U; - src = *(in + 32 * 26 + i); - tmp |= src << 14U; - src = *(in + 32 * 27 + i); - tmp |= src << 17U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 23U; - src = *(in + 32 * 30 + i); - tmp |= src << 26U; - src = *(in + 32 * 31 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out -= 64; - } -} -void pack_4bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 4U; - src = *(in + 32 * 2 + i); - tmp |= src << 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 12U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 20U; - src = *(in + 32 * 6 + i); - tmp |= src << 24U; - src = *(in + 32 * 7 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 4U; - src = *(in + 32 * 10 + i); - tmp |= src << 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 12U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 20U; - src = *(in + 32 * 14 + i); - tmp |= src << 24U; - src = *(in + 32 * 15 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 4U; - src = *(in + 32 * 18 + i); - tmp |= src << 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 12U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 20U; - src = *(in + 32 * 22 + i); - tmp |= src << 24U; - src = *(in + 32 * 23 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 4U; - src = *(in + 32 * 26 + i); - tmp |= src << 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 12U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 20U; - src = *(in + 32 * 30 + i); - tmp |= src << 24U; - src = *(in + 32 * 31 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out -= 96; - } -} -void pack_5bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 5U; - src = *(in + 32 * 2 + i); - tmp |= src << 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 15U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - src = *(in + 32 * 5 + i); - tmp |= src << 25U; - src = *(in + 32 * 6 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 3U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 13U; - src = *(in + 32 * 10 + i); - tmp |= src << 18U; - src = *(in + 32 * 11 + i); - tmp |= src << 23U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 1U; - src = *(in + 32 * 14 + i); - tmp |= src << 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 11U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 21U; - src = *(in + 32 * 18 + i); - tmp |= src << 26U; - src = *(in + 32 * 19 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 1U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 9U; - src = *(in + 32 * 22 + i); - tmp |= src << 14U; - src = *(in + 32 * 23 + i); - tmp |= src << 19U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 3U; - src = *(in + 32 * 26 + i); - tmp |= src << 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 7U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 17U; - src = *(in + 32 * 30 + i); - tmp |= src << 22U; - src = *(in + 32 * 31 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_6bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 6U; - src = *(in + 32 * 2 + i); - tmp |= src << 12U; - src = *(in + 32 * 3 + i); - tmp |= src << 18U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - src = *(in + 32 * 5 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 2U; - src = *(in + 32 * 6 + i); - tmp |= src << 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 10U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 22U; - src = *(in + 32 * 10 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 2U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 14U; - src = *(in + 32 * 14 + i); - tmp |= src << 20U; - src = *(in + 32 * 15 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 6U; - src = *(in + 32 * 18 + i); - tmp |= src << 12U; - src = *(in + 32 * 19 + i); - tmp |= src << 18U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - src = *(in + 32 * 21 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 2U; - src = *(in + 32 * 22 + i); - tmp |= src << 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 10U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 22U; - src = *(in + 32 * 26 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 2U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 14U; - src = *(in + 32 * 30 + i); - tmp |= src << 20U; - src = *(in + 32 * 31 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out -= 160; - } -} -void pack_7bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 7U; - src = *(in + 32 * 2 + i); - tmp |= src << 14U; - src = *(in + 32 * 3 + i); - tmp |= src << 21U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 3U; - src = *(in + 32 * 6 + i); - tmp |= src << 10U; - src = *(in + 32 * 7 + i); - tmp |= src << 17U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 1U; - src = *(in + 32 * 10 + i); - tmp |= src << 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 13U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - src = *(in + 32 * 13 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 5U; - src = *(in + 32 * 14 + i); - tmp |= src << 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 9U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 23U; - src = *(in + 32 * 18 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 5U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 19U; - src = *(in + 32 * 22 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 1U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 15U; - src = *(in + 32 * 26 + i); - tmp |= src << 22U; - src = *(in + 32 * 27 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 3U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 11U; - src = *(in + 32 * 30 + i); - tmp |= src << 18U; - src = *(in + 32 * 31 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out -= 192; - } -} -void pack_8bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 8U; - src = *(in + 32 * 2 + i); - tmp |= src << 16U; - src = *(in + 32 * 3 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - src = *(in + 32 * 5 + i); - tmp |= src << 8U; - src = *(in + 32 * 6 + i); - tmp |= src << 16U; - src = *(in + 32 * 7 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 8U; - src = *(in + 32 * 10 + i); - tmp |= src << 16U; - src = *(in + 32 * 11 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - src = *(in + 32 * 13 + i); - tmp |= src << 8U; - src = *(in + 32 * 14 + i); - tmp |= src << 16U; - src = *(in + 32 * 15 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 8U; - src = *(in + 32 * 18 + i); - tmp |= src << 16U; - src = *(in + 32 * 19 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - src = *(in + 32 * 21 + i); - tmp |= src << 8U; - src = *(in + 32 * 22 + i); - tmp |= src << 16U; - src = *(in + 32 * 23 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 8U; - src = *(in + 32 * 26 + i); - tmp |= src << 16U; - src = *(in + 32 * 27 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - src = *(in + 32 * 29 + i); - tmp |= src << 8U; - src = *(in + 32 * 30 + i); - tmp |= src << 16U; - src = *(in + 32 * 31 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out -= 224; - } -} -void pack_9bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 9U; - src = *(in + 32 * 2 + i); - tmp |= src << 18U; - src = *(in + 32 * 3 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 5U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 13U; - src = *(in + 32 * 6 + i); - tmp |= src << 22U; - src = *(in + 32 * 7 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 1U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 17U; - src = *(in + 32 * 10 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 3U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 21U; - src = *(in + 32 * 14 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 7U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 7U; - src = *(in + 32 * 18 + i); - tmp |= src << 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 11U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 3U; - src = *(in + 32 * 22 + i); - tmp |= src << 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 15U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 1U; - src = *(in + 32 * 26 + i); - tmp |= src << 10U; - src = *(in + 32 * 27 + i); - tmp |= src << 19U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 5U; - src = *(in + 32 * 30 + i); - tmp |= src << 14U; - src = *(in + 32 * 31 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_10bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 10U; - src = *(in + 32 * 2 + i); - tmp |= src << 20U; - src = *(in + 32 * 3 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 2U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 18U; - src = *(in + 32 * 6 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 6U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 6U; - src = *(in + 32 * 10 + i); - tmp |= src << 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 14U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 2U; - src = *(in + 32 * 14 + i); - tmp |= src << 12U; - src = *(in + 32 * 15 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 10U; - src = *(in + 32 * 18 + i); - tmp |= src << 20U; - src = *(in + 32 * 19 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 2U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 18U; - src = *(in + 32 * 22 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 6U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 6U; - src = *(in + 32 * 26 + i); - tmp |= src << 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 14U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 2U; - src = *(in + 32 * 30 + i); - tmp |= src << 12U; - src = *(in + 32 * 31 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out -= 288; - } -} -void pack_11bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 11U; - src = *(in + 32 * 2 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 1U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 9U; - src = *(in + 32 * 6 + i); - tmp |= src << 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 13U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 3U; - src = *(in + 32 * 10 + i); - tmp |= src << 14U; - src = *(in + 32 * 11 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 7U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 15U; - src = *(in + 32 * 14 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 5U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 5U; - src = *(in + 32 * 18 + i); - tmp |= src << 6U; - src = *(in + 32 * 19 + i); - tmp |= src << 17U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 7U; - src = *(in + 32 * 22 + i); - tmp |= src << 18U; - src = *(in + 32 * 23 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 3U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 19U; - src = *(in + 32 * 26 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 9U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 1U; - src = *(in + 32 * 30 + i); - tmp |= src << 10U; - src = *(in + 32 * 31 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out -= 320; - } -} -void pack_12bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 12U; - src = *(in + 32 * 2 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 4U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 4U; - src = *(in + 32 * 6 + i); - tmp |= src << 8U; - src = *(in + 32 * 7 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 12U; - src = *(in + 32 * 10 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 4U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 4U; - src = *(in + 32 * 14 + i); - tmp |= src << 8U; - src = *(in + 32 * 15 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 12U; - src = *(in + 32 * 18 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 4U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 4U; - src = *(in + 32 * 22 + i); - tmp |= src << 8U; - src = *(in + 32 * 23 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 12U; - src = *(in + 32 * 26 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 4U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 4U; - src = *(in + 32 * 30 + i); - tmp |= src << 8U; - src = *(in + 32 * 31 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out -= 352; - } -} -void pack_13bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 13U; - src = *(in + 32 * 2 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 7U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 1U; - src = *(in + 32 * 6 + i); - tmp |= src << 14U; - src = *(in + 32 * 7 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 5U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 11U; - src = *(in + 32 * 10 + i); - tmp |= src << 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 15U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 9U; - src = *(in + 32 * 14 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 3U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 3U; - src = *(in + 32 * 18 + i); - tmp |= src << 10U; - src = *(in + 32 * 19 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 9U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 17U; - src = *(in + 32 * 22 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 11U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 5U; - src = *(in + 32 * 26 + i); - tmp |= src << 18U; - src = *(in + 32 * 27 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 1U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 7U; - src = *(in + 32 * 30 + i); - tmp |= src << 6U; - src = *(in + 32 * 31 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_14bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 14U; - src = *(in + 32 * 2 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 10U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 6U; - src = *(in + 32 * 6 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 2U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 2U; - src = *(in + 32 * 10 + i); - tmp |= src << 12U; - src = *(in + 32 * 11 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 6U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 10U; - src = *(in + 32 * 14 + i); - tmp |= src << 4U; - src = *(in + 32 * 15 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 14U; - src = *(in + 32 * 18 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 10U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 6U; - src = *(in + 32 * 22 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 2U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 2U; - src = *(in + 32 * 26 + i); - tmp |= src << 12U; - src = *(in + 32 * 27 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 6U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 10U; - src = *(in + 32 * 30 + i); - tmp |= src << 4U; - src = *(in + 32 * 31 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out -= 416; - } -} -void pack_15bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 15U; - src = *(in + 32 * 2 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 13U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 11U; - src = *(in + 32 * 6 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 9U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 7U; - src = *(in + 32 * 10 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 5U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 3U; - src = *(in + 32 * 14 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 1U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 1U; - src = *(in + 32 * 18 + i); - tmp |= src << 14U; - src = *(in + 32 * 19 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 3U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 5U; - src = *(in + 32 * 22 + i); - tmp |= src << 10U; - src = *(in + 32 * 23 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 7U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 9U; - src = *(in + 32 * 26 + i); - tmp |= src << 6U; - src = *(in + 32 * 27 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 11U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 13U; - src = *(in + 32 * 30 + i); - tmp |= src << 2U; - src = *(in + 32 * 31 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out -= 448; - } -} -void pack_16bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src; - src = *(in + 32 * 3 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - src = *(in + 32 * 5 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src; - src = *(in + 32 * 7 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src; - src = *(in + 32 * 11 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - src = *(in + 32 * 13 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src; - src = *(in + 32 * 15 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src; - src = *(in + 32 * 19 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - src = *(in + 32 * 21 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src; - src = *(in + 32 * 23 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src; - src = *(in + 32 * 27 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - src = *(in + 32 * 29 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src; - src = *(in + 32 * 31 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out -= 480; - } -} -void pack_17bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 15U; - src = *(in + 32 * 2 + i); - tmp |= src << 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 13U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 11U; - src = *(in + 32 * 6 + i); - tmp |= src << 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 9U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 7U; - src = *(in + 32 * 10 + i); - tmp |= src << 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 5U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 3U; - src = *(in + 32 * 14 + i); - tmp |= src << 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 1U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 1U; - src = *(in + 32 * 18 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 14U; - src = *(in + 32 * 19 + i); - tmp |= src << 3U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 5U; - src = *(in + 32 * 22 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 10U; - src = *(in + 32 * 23 + i); - tmp |= src << 7U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 9U; - src = *(in + 32 * 26 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 6U; - src = *(in + 32 * 27 + i); - tmp |= src << 11U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 13U; - src = *(in + 32 * 30 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 2U; - src = *(in + 32 * 31 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_18bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 14U; - src = *(in + 32 * 2 + i); - tmp |= src << 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 10U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 6U; - src = *(in + 32 * 6 + i); - tmp |= src << 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 2U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 2U; - src = *(in + 32 * 10 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 12U; - src = *(in + 32 * 11 + i); - tmp |= src << 6U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 10U; - src = *(in + 32 * 14 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 4U; - src = *(in + 32 * 15 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 14U; - src = *(in + 32 * 18 + i); - tmp |= src << 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 10U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 6U; - src = *(in + 32 * 22 + i); - tmp |= src << 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 2U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 2U; - src = *(in + 32 * 26 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 12U; - src = *(in + 32 * 27 + i); - tmp |= src << 6U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 10U; - src = *(in + 32 * 30 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 4U; - src = *(in + 32 * 31 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out -= 544; - } -} -void pack_19bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 13U; - src = *(in + 32 * 2 + i); - tmp |= src << 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 7U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 1U; - src = *(in + 32 * 6 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 14U; - src = *(in + 32 * 7 + i); - tmp |= src << 5U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 11U; - src = *(in + 32 * 10 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 15U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 9U; - src = *(in + 32 * 14 + i); - tmp |= src << 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 3U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 3U; - src = *(in + 32 * 18 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 10U; - src = *(in + 32 * 19 + i); - tmp |= src << 9U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 17U; - src = *(in + 32 * 22 + i); - tmp |= src << 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 11U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 5U; - src = *(in + 32 * 26 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 18U; - src = *(in + 32 * 27 + i); - tmp |= src << 1U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 7U; - src = *(in + 32 * 30 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 6U; - src = *(in + 32 * 31 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out -= 576; - } -} -void pack_20bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 12U; - src = *(in + 32 * 2 + i); - tmp |= src << 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 4U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 4U; - src = *(in + 32 * 6 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 8U; - src = *(in + 32 * 7 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 12U; - src = *(in + 32 * 10 + i); - tmp |= src << 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 4U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 4U; - src = *(in + 32 * 14 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 8U; - src = *(in + 32 * 15 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 12U; - src = *(in + 32 * 18 + i); - tmp |= src << 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 4U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 4U; - src = *(in + 32 * 22 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 8U; - src = *(in + 32 * 23 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 12U; - src = *(in + 32 * 26 + i); - tmp |= src << 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 4U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 4U; - src = *(in + 32 * 30 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 8U; - src = *(in + 32 * 31 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out -= 608; - } -} -void pack_21bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 11U; - src = *(in + 32 * 2 + i); - tmp |= src << 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 1U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 9U; - src = *(in + 32 * 6 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 13U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 3U; - src = *(in + 32 * 10 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 14U; - src = *(in + 32 * 11 + i); - tmp |= src << 7U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 15U; - src = *(in + 32 * 14 + i); - tmp |= src << 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 5U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 5U; - src = *(in + 32 * 18 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 6U; - src = *(in + 32 * 19 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 17U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 7U; - src = *(in + 32 * 22 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 18U; - src = *(in + 32 * 23 + i); - tmp |= src << 3U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 19U; - src = *(in + 32 * 26 + i); - tmp |= src << 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 9U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 1U; - src = *(in + 32 * 30 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 10U; - src = *(in + 32 * 31 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_22bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 10U; - src = *(in + 32 * 2 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 20U; - src = *(in + 32 * 3 + i); - tmp |= src << 2U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 18U; - src = *(in + 32 * 6 + i); - tmp |= src << 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 6U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 6U; - src = *(in + 32 * 10 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 14U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 2U; - src = *(in + 32 * 14 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 12U; - src = *(in + 32 * 15 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 10U; - src = *(in + 32 * 18 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 20U; - src = *(in + 32 * 19 + i); - tmp |= src << 2U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 18U; - src = *(in + 32 * 22 + i); - tmp |= src << 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 6U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 6U; - src = *(in + 32 * 26 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 14U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 2U; - src = *(in + 32 * 30 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 12U; - src = *(in + 32 * 31 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out -= 672; - } -} -void pack_23bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 9U; - src = *(in + 32 * 2 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 18U; - src = *(in + 32 * 3 + i); - tmp |= src << 5U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 13U; - src = *(in + 32 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 22U; - src = *(in + 32 * 7 + i); - tmp |= src << 1U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 17U; - src = *(in + 32 * 10 + i); - tmp |= src << 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 3U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 21U; - src = *(in + 32 * 14 + i); - tmp |= src << 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 7U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 7U; - src = *(in + 32 * 18 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 11U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 3U; - src = *(in + 32 * 22 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 15U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 1U; - src = *(in + 32 * 26 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 10U; - src = *(in + 32 * 27 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 19U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 5U; - src = *(in + 32 * 30 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 14U; - src = *(in + 32 * 31 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out -= 704; - } -} -void pack_24bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 8U; - src = *(in + 32 * 2 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 16U; - src = *(in + 32 * 3 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - src = *(in + 32 * 5 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 8U; - src = *(in + 32 * 6 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 16U; - src = *(in + 32 * 7 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 8U; - src = *(in + 32 * 10 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 16U; - src = *(in + 32 * 11 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - src = *(in + 32 * 13 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 8U; - src = *(in + 32 * 14 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 16U; - src = *(in + 32 * 15 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 8U; - src = *(in + 32 * 18 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 16U; - src = *(in + 32 * 19 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - src = *(in + 32 * 21 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 8U; - src = *(in + 32 * 22 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 16U; - src = *(in + 32 * 23 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 8U; - src = *(in + 32 * 26 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 16U; - src = *(in + 32 * 27 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - src = *(in + 32 * 29 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 8U; - src = *(in + 32 * 30 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 16U; - src = *(in + 32 * 31 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out -= 736; - } -} -void pack_25bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 7U; - src = *(in + 32 * 2 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 14U; - src = *(in + 32 * 3 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 21U; - src = *(in + 32 * 4 + i); - tmp |= src << 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 3U; - src = *(in + 32 * 6 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 10U; - src = *(in + 32 * 7 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 17U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 1U; - src = *(in + 32 * 10 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 6U; - src = *(in + 32 * 11 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 13U; - src = *(in + 32 * 12 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 20U; - src = *(in + 32 * 13 + i); - tmp |= src << 5U; - src = *(in + 32 * 14 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 2U; - src = *(in + 32 * 15 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 9U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 23U; - src = *(in + 32 * 18 + i); - tmp |= src << 2U; - src = *(in + 32 * 19 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 5U; - src = *(in + 32 * 20 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 12U; - src = *(in + 32 * 21 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 19U; - src = *(in + 32 * 22 + i); - tmp |= src << 6U; - src = *(in + 32 * 23 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 1U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 15U; - src = *(in + 32 * 26 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 22U; - src = *(in + 32 * 27 + i); - tmp |= src << 3U; - src = *(in + 32 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 4U; - src = *(in + 32 * 29 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 11U; - src = *(in + 32 * 30 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 18U; - src = *(in + 32 * 31 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_26bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 6U; - src = *(in + 32 * 2 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 12U; - src = *(in + 32 * 3 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 18U; - src = *(in + 32 * 4 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 24U; - src = *(in + 32 * 5 + i); - tmp |= src << 2U; - src = *(in + 32 * 6 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 4U; - src = *(in + 32 * 7 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 10U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 22U; - src = *(in + 32 * 10 + i); - tmp |= src << 4U; - src = *(in + 32 * 11 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 2U; - src = *(in + 32 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 8U; - src = *(in + 32 * 13 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 14U; - src = *(in + 32 * 14 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 20U; - src = *(in + 32 * 15 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 6U; - src = *(in + 32 * 18 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 12U; - src = *(in + 32 * 19 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 18U; - src = *(in + 32 * 20 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 24U; - src = *(in + 32 * 21 + i); - tmp |= src << 2U; - src = *(in + 32 * 22 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 4U; - src = *(in + 32 * 23 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 10U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 22U; - src = *(in + 32 * 26 + i); - tmp |= src << 4U; - src = *(in + 32 * 27 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 2U; - src = *(in + 32 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 8U; - src = *(in + 32 * 29 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 14U; - src = *(in + 32 * 30 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 20U; - src = *(in + 32 * 31 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 800; - } -} -void pack_27bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 5U; - src = *(in + 32 * 2 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 10U; - src = *(in + 32 * 3 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 15U; - src = *(in + 32 * 4 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 20U; - src = *(in + 32 * 5 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 25U; - src = *(in + 32 * 6 + i); - tmp |= src << 2U; - src = *(in + 32 * 7 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 3U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 13U; - src = *(in + 32 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 18U; - src = *(in + 32 * 11 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 23U; - src = *(in + 32 * 12 + i); - tmp |= src << 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 1U; - src = *(in + 32 * 14 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 6U; - src = *(in + 32 * 15 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 11U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 21U; - src = *(in + 32 * 18 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 26U; - src = *(in + 32 * 19 + i); - tmp |= src << 1U; - src = *(in + 32 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 4U; - src = *(in + 32 * 21 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 9U; - src = *(in + 32 * 22 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 14U; - src = *(in + 32 * 23 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 19U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 3U; - src = *(in + 32 * 26 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 2U; - src = *(in + 32 * 27 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 7U; - src = *(in + 32 * 28 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 12U; - src = *(in + 32 * 29 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 17U; - src = *(in + 32 * 30 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 22U; - src = *(in + 32 * 31 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 832; - } -} -void pack_28bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 4U; - src = *(in + 32 * 2 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 8U; - src = *(in + 32 * 3 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 12U; - src = *(in + 32 * 4 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 16U; - src = *(in + 32 * 5 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 20U; - src = *(in + 32 * 6 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 24U; - src = *(in + 32 * 7 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - src = *(in + 32 * 9 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 4U; - src = *(in + 32 * 10 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 8U; - src = *(in + 32 * 11 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 12U; - src = *(in + 32 * 12 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 16U; - src = *(in + 32 * 13 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 20U; - src = *(in + 32 * 14 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 24U; - src = *(in + 32 * 15 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 4U; - src = *(in + 32 * 18 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 8U; - src = *(in + 32 * 19 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 12U; - src = *(in + 32 * 20 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 16U; - src = *(in + 32 * 21 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 20U; - src = *(in + 32 * 22 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 24U; - src = *(in + 32 * 23 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - src = *(in + 32 * 25 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 4U; - src = *(in + 32 * 26 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 8U; - src = *(in + 32 * 27 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 12U; - src = *(in + 32 * 28 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 16U; - src = *(in + 32 * 29 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 20U; - src = *(in + 32 * 30 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 24U; - src = *(in + 32 * 31 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 864; - } -} -void pack_29bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 3U; - src = *(in + 32 * 2 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 6U; - src = *(in + 32 * 3 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 9U; - src = *(in + 32 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 12U; - src = *(in + 32 * 5 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 15U; - src = *(in + 32 * 6 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 18U; - src = *(in + 32 * 7 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 21U; - src = *(in + 32 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 24U; - src = *(in + 32 * 9 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 27U; - src = *(in + 32 * 10 + i); - tmp |= src << 2U; - src = *(in + 32 * 11 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 1U; - src = *(in + 32 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 4U; - src = *(in + 32 * 13 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 7U; - src = *(in + 32 * 14 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 10U; - src = *(in + 32 * 15 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 13U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 19U; - src = *(in + 32 * 18 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 22U; - src = *(in + 32 * 19 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 25U; - src = *(in + 32 * 20 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 28U; - src = *(in + 32 * 21 + i); - tmp |= src << 1U; - src = *(in + 32 * 22 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 2U; - src = *(in + 32 * 23 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 5U; - src = *(in + 32 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 8U; - src = *(in + 32 * 25 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 11U; - src = *(in + 32 * 26 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 14U; - src = *(in + 32 * 27 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 17U; - src = *(in + 32 * 28 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 20U; - src = *(in + 32 * 29 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 23U; - src = *(in + 32 * 30 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 26U; - src = *(in + 32 * 31 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 896; - } -} -void pack_30bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 2U; - src = *(in + 32 * 2 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 4U; - src = *(in + 32 * 3 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 6U; - src = *(in + 32 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 8U; - src = *(in + 32 * 5 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 10U; - src = *(in + 32 * 6 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 12U; - src = *(in + 32 * 7 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 14U; - src = *(in + 32 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 16U; - src = *(in + 32 * 9 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 18U; - src = *(in + 32 * 10 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 20U; - src = *(in + 32 * 11 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 22U; - src = *(in + 32 * 12 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 24U; - src = *(in + 32 * 13 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 26U; - src = *(in + 32 * 14 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 28U; - src = *(in + 32 * 15 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - src = *(in + 32 * 17 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 2U; - src = *(in + 32 * 18 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 4U; - src = *(in + 32 * 19 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 6U; - src = *(in + 32 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 8U; - src = *(in + 32 * 21 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 10U; - src = *(in + 32 * 22 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 12U; - src = *(in + 32 * 23 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 14U; - src = *(in + 32 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 16U; - src = *(in + 32 * 25 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 18U; - src = *(in + 32 * 26 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 20U; - src = *(in + 32 * 27 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 22U; - src = *(in + 32 * 28 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 24U; - src = *(in + 32 * 29 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 26U; - src = *(in + 32 * 30 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 28U; - src = *(in + 32 * 31 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 928; - } -} -void pack_31bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - src = *(in + 32 * 1 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src >> 1U; - src = *(in + 32 * 2 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src >> 2U; - src = *(in + 32 * 3 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src >> 3U; - src = *(in + 32 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src >> 4U; - src = *(in + 32 * 5 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src >> 5U; - src = *(in + 32 * 6 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src >> 6U; - src = *(in + 32 * 7 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src >> 7U; - src = *(in + 32 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src >> 8U; - src = *(in + 32 * 9 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src >> 9U; - src = *(in + 32 * 10 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src >> 10U; - src = *(in + 32 * 11 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src >> 11U; - src = *(in + 32 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src >> 12U; - src = *(in + 32 * 13 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src >> 13U; - src = *(in + 32 * 14 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src >> 14U; - src = *(in + 32 * 15 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src >> 15U; - src = *(in + 32 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src >> 16U; - src = *(in + 32 * 17 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src >> 17U; - src = *(in + 32 * 18 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src >> 18U; - src = *(in + 32 * 19 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src >> 19U; - src = *(in + 32 * 20 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src >> 20U; - src = *(in + 32 * 21 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src >> 21U; - src = *(in + 32 * 22 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src >> 22U; - src = *(in + 32 * 23 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src >> 23U; - src = *(in + 32 * 24 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src >> 24U; - src = *(in + 32 * 25 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src >> 25U; - src = *(in + 32 * 26 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src >> 26U; - src = *(in + 32 * 27 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src >> 27U; - src = *(in + 32 * 28 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src >> 28U; - src = *(in + 32 * 29 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src >> 29U; - src = *(in + 32 * 30 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src >> 30U; - src = *(in + 32 * 31 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 960; - } -} -void pack_32bit_32simdWidth(uint32_t* __restrict in, uint32_t* __restrict out) { - uint32_t tmp = 0U; - uint32_t src; - for (int i = 0; i < 32; i++) { - src = *(in + 32 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 7 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 8 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 9 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 10 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 11 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 12 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 13 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 14 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 15 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 16 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 17 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 18 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 19 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 20 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 21 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 22 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 23 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 24 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 25 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 26 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 27 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 28 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 29 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 30 + i); - tmp = src; - *(out + i) = tmp; - out += 32; - src = *(in + 32 * 31 + i); - tmp = src; - *(out + i) = tmp; - out -= 992; - } -} -void pack_1bit_16simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 1U; - src = *(in + 16 * 2 + i); - tmp |= src << 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 3U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 5U; - src = *(in + 16 * 6 + i); - tmp |= src << 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 7U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 9U; - src = *(in + 16 * 10 + i); - tmp |= src << 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 11U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 13U; - src = *(in + 16 * 14 + i); - tmp |= src << 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 15U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 17U; - src = *(in + 16 * 18 + i); - tmp |= src << 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 19U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 21U; - src = *(in + 16 * 22 + i); - tmp |= src << 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 23U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 25U; - src = *(in + 16 * 26 + i); - tmp |= src << 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 27U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 29U; - src = *(in + 16 * 30 + i); - tmp |= src << 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 31U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 33U; - src = *(in + 16 * 34 + i); - tmp |= src << 34U; - src = *(in + 16 * 35 + i); - tmp |= src << 35U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 37U; - src = *(in + 16 * 38 + i); - tmp |= src << 38U; - src = *(in + 16 * 39 + i); - tmp |= src << 39U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 41U; - src = *(in + 16 * 42 + i); - tmp |= src << 42U; - src = *(in + 16 * 43 + i); - tmp |= src << 43U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 45U; - src = *(in + 16 * 46 + i); - tmp |= src << 46U; - src = *(in + 16 * 47 + i); - tmp |= src << 47U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 49U; - src = *(in + 16 * 50 + i); - tmp |= src << 50U; - src = *(in + 16 * 51 + i); - tmp |= src << 51U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - src = *(in + 16 * 53 + i); - tmp |= src << 53U; - src = *(in + 16 * 54 + i); - tmp |= src << 54U; - src = *(in + 16 * 55 + i); - tmp |= src << 55U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - src = *(in + 16 * 57 + i); - tmp |= src << 57U; - src = *(in + 16 * 58 + i); - tmp |= src << 58U; - src = *(in + 16 * 59 + i); - tmp |= src << 59U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - src = *(in + 16 * 61 + i); - tmp |= src << 61U; - src = *(in + 16 * 62 + i); - tmp |= src << 62U; - src = *(in + 16 * 63 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out -= 0; - } -} -void pack_2bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 2U; - src = *(in + 16 * 2 + i); - tmp |= src << 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 6U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 10U; - src = *(in + 16 * 6 + i); - tmp |= src << 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 14U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 18U; - src = *(in + 16 * 10 + i); - tmp |= src << 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 22U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 26U; - src = *(in + 16 * 14 + i); - tmp |= src << 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 30U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 34U; - src = *(in + 16 * 18 + i); - tmp |= src << 36U; - src = *(in + 16 * 19 + i); - tmp |= src << 38U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 42U; - src = *(in + 16 * 22 + i); - tmp |= src << 44U; - src = *(in + 16 * 23 + i); - tmp |= src << 46U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 50U; - src = *(in + 16 * 26 + i); - tmp |= src << 52U; - src = *(in + 16 * 27 + i); - tmp |= src << 54U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - src = *(in + 16 * 29 + i); - tmp |= src << 58U; - src = *(in + 16 * 30 + i); - tmp |= src << 60U; - src = *(in + 16 * 31 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 2U; - src = *(in + 16 * 34 + i); - tmp |= src << 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 6U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 10U; - src = *(in + 16 * 38 + i); - tmp |= src << 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 14U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 18U; - src = *(in + 16 * 42 + i); - tmp |= src << 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 22U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 26U; - src = *(in + 16 * 46 + i); - tmp |= src << 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 30U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 34U; - src = *(in + 16 * 50 + i); - tmp |= src << 36U; - src = *(in + 16 * 51 + i); - tmp |= src << 38U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 42U; - src = *(in + 16 * 54 + i); - tmp |= src << 44U; - src = *(in + 16 * 55 + i); - tmp |= src << 46U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 50U; - src = *(in + 16 * 58 + i); - tmp |= src << 52U; - src = *(in + 16 * 59 + i); - tmp |= src << 54U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - src = *(in + 16 * 61 + i); - tmp |= src << 58U; - src = *(in + 16 * 62 + i); - tmp |= src << 60U; - src = *(in + 16 * 63 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out -= 16; - } -} -void pack_3bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 3U; - src = *(in + 16 * 2 + i); - tmp |= src << 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 9U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 15U; - src = *(in + 16 * 6 + i); - tmp |= src << 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 21U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 27U; - src = *(in + 16 * 10 + i); - tmp |= src << 30U; - src = *(in + 16 * 11 + i); - tmp |= src << 33U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 39U; - src = *(in + 16 * 14 + i); - tmp |= src << 42U; - src = *(in + 16 * 15 + i); - tmp |= src << 45U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 51U; - src = *(in + 16 * 18 + i); - tmp |= src << 54U; - src = *(in + 16 * 19 + i); - tmp |= src << 57U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - src = *(in + 16 * 21 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 1U; - src = *(in + 16 * 22 + i); - tmp |= src << 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 5U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 11U; - src = *(in + 16 * 26 + i); - tmp |= src << 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 17U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 23U; - src = *(in + 16 * 30 + i); - tmp |= src << 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 29U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 35U; - src = *(in + 16 * 34 + i); - tmp |= src << 38U; - src = *(in + 16 * 35 + i); - tmp |= src << 41U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 47U; - src = *(in + 16 * 38 + i); - tmp |= src << 50U; - src = *(in + 16 * 39 + i); - tmp |= src << 53U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - src = *(in + 16 * 41 + i); - tmp |= src << 59U; - src = *(in + 16 * 42 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 1U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 7U; - src = *(in + 16 * 46 + i); - tmp |= src << 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 13U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 19U; - src = *(in + 16 * 50 + i); - tmp |= src << 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 25U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 31U; - src = *(in + 16 * 54 + i); - tmp |= src << 34U; - src = *(in + 16 * 55 + i); - tmp |= src << 37U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 43U; - src = *(in + 16 * 58 + i); - tmp |= src << 46U; - src = *(in + 16 * 59 + i); - tmp |= src << 49U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - src = *(in + 16 * 61 + i); - tmp |= src << 55U; - src = *(in + 16 * 62 + i); - tmp |= src << 58U; - src = *(in + 16 * 63 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out -= 32; - } -} -void pack_4bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 4U; - src = *(in + 16 * 2 + i); - tmp |= src << 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 12U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 20U; - src = *(in + 16 * 6 + i); - tmp |= src << 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 28U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 36U; - src = *(in + 16 * 10 + i); - tmp |= src << 40U; - src = *(in + 16 * 11 + i); - tmp |= src << 44U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - src = *(in + 16 * 13 + i); - tmp |= src << 52U; - src = *(in + 16 * 14 + i); - tmp |= src << 56U; - src = *(in + 16 * 15 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 4U; - src = *(in + 16 * 18 + i); - tmp |= src << 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 12U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 20U; - src = *(in + 16 * 22 + i); - tmp |= src << 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 28U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 36U; - src = *(in + 16 * 26 + i); - tmp |= src << 40U; - src = *(in + 16 * 27 + i); - tmp |= src << 44U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - src = *(in + 16 * 29 + i); - tmp |= src << 52U; - src = *(in + 16 * 30 + i); - tmp |= src << 56U; - src = *(in + 16 * 31 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 4U; - src = *(in + 16 * 34 + i); - tmp |= src << 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 12U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 20U; - src = *(in + 16 * 38 + i); - tmp |= src << 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 28U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 36U; - src = *(in + 16 * 42 + i); - tmp |= src << 40U; - src = *(in + 16 * 43 + i); - tmp |= src << 44U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - src = *(in + 16 * 45 + i); - tmp |= src << 52U; - src = *(in + 16 * 46 + i); - tmp |= src << 56U; - src = *(in + 16 * 47 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 4U; - src = *(in + 16 * 50 + i); - tmp |= src << 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 12U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 20U; - src = *(in + 16 * 54 + i); - tmp |= src << 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 28U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 36U; - src = *(in + 16 * 58 + i); - tmp |= src << 40U; - src = *(in + 16 * 59 + i); - tmp |= src << 44U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - src = *(in + 16 * 61 + i); - tmp |= src << 52U; - src = *(in + 16 * 62 + i); - tmp |= src << 56U; - src = *(in + 16 * 63 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out -= 48; - } -} -void pack_5bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 5U; - src = *(in + 16 * 2 + i); - tmp |= src << 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 15U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 25U; - src = *(in + 16 * 6 + i); - tmp |= src << 30U; - src = *(in + 16 * 7 + i); - tmp |= src << 35U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 45U; - src = *(in + 16 * 10 + i); - tmp |= src << 50U; - src = *(in + 16 * 11 + i); - tmp |= src << 55U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 1U; - src = *(in + 16 * 14 + i); - tmp |= src << 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 11U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 21U; - src = *(in + 16 * 18 + i); - tmp |= src << 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 31U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 41U; - src = *(in + 16 * 22 + i); - tmp |= src << 46U; - src = *(in + 16 * 23 + i); - tmp |= src << 51U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - src = *(in + 16 * 25 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 3U; - src = *(in + 16 * 26 + i); - tmp |= src << 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 7U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 17U; - src = *(in + 16 * 30 + i); - tmp |= src << 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 27U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 37U; - src = *(in + 16 * 34 + i); - tmp |= src << 42U; - src = *(in + 16 * 35 + i); - tmp |= src << 47U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - src = *(in + 16 * 37 + i); - tmp |= src << 57U; - src = *(in + 16 * 38 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 3U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 13U; - src = *(in + 16 * 42 + i); - tmp |= src << 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 23U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 33U; - src = *(in + 16 * 46 + i); - tmp |= src << 38U; - src = *(in + 16 * 47 + i); - tmp |= src << 43U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 53U; - src = *(in + 16 * 50 + i); - tmp |= src << 58U; - src = *(in + 16 * 51 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 1U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 9U; - src = *(in + 16 * 54 + i); - tmp |= src << 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 19U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 29U; - src = *(in + 16 * 58 + i); - tmp |= src << 34U; - src = *(in + 16 * 59 + i); - tmp |= src << 39U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - src = *(in + 16 * 61 + i); - tmp |= src << 49U; - src = *(in + 16 * 62 + i); - tmp |= src << 54U; - src = *(in + 16 * 63 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out -= 64; - } -} -void pack_6bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 6U; - src = *(in + 16 * 2 + i); - tmp |= src << 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 18U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 30U; - src = *(in + 16 * 6 + i); - tmp |= src << 36U; - src = *(in + 16 * 7 + i); - tmp |= src << 42U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 54U; - src = *(in + 16 * 10 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 2U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 14U; - src = *(in + 16 * 14 + i); - tmp |= src << 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 26U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 38U; - src = *(in + 16 * 18 + i); - tmp |= src << 44U; - src = *(in + 16 * 19 + i); - tmp |= src << 50U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - src = *(in + 16 * 21 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 2U; - src = *(in + 16 * 22 + i); - tmp |= src << 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 10U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 22U; - src = *(in + 16 * 26 + i); - tmp |= src << 28U; - src = *(in + 16 * 27 + i); - tmp |= src << 34U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 46U; - src = *(in + 16 * 30 + i); - tmp |= src << 52U; - src = *(in + 16 * 31 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 6U; - src = *(in + 16 * 34 + i); - tmp |= src << 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 18U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 30U; - src = *(in + 16 * 38 + i); - tmp |= src << 36U; - src = *(in + 16 * 39 + i); - tmp |= src << 42U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 54U; - src = *(in + 16 * 42 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 2U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 14U; - src = *(in + 16 * 46 + i); - tmp |= src << 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 26U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 38U; - src = *(in + 16 * 50 + i); - tmp |= src << 44U; - src = *(in + 16 * 51 + i); - tmp |= src << 50U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - src = *(in + 16 * 53 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 2U; - src = *(in + 16 * 54 + i); - tmp |= src << 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 10U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 22U; - src = *(in + 16 * 58 + i); - tmp |= src << 28U; - src = *(in + 16 * 59 + i); - tmp |= src << 34U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 46U; - src = *(in + 16 * 62 + i); - tmp |= src << 52U; - src = *(in + 16 * 63 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out -= 80; - } -} -void pack_7bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 7U; - src = *(in + 16 * 2 + i); - tmp |= src << 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 21U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 35U; - src = *(in + 16 * 6 + i); - tmp |= src << 42U; - src = *(in + 16 * 7 + i); - tmp |= src << 49U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - src = *(in + 16 * 9 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 1U; - src = *(in + 16 * 10 + i); - tmp |= src << 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 13U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 27U; - src = *(in + 16 * 14 + i); - tmp |= src << 34U; - src = *(in + 16 * 15 + i); - tmp |= src << 41U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 55U; - src = *(in + 16 * 18 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 5U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 19U; - src = *(in + 16 * 22 + i); - tmp |= src << 26U; - src = *(in + 16 * 23 + i); - tmp |= src << 33U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 47U; - src = *(in + 16 * 26 + i); - tmp |= src << 54U; - src = *(in + 16 * 27 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 3U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 11U; - src = *(in + 16 * 30 + i); - tmp |= src << 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 25U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 39U; - src = *(in + 16 * 34 + i); - tmp |= src << 46U; - src = *(in + 16 * 35 + i); - tmp |= src << 53U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 3U; - src = *(in + 16 * 38 + i); - tmp |= src << 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 17U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 31U; - src = *(in + 16 * 42 + i); - tmp |= src << 38U; - src = *(in + 16 * 43 + i); - tmp |= src << 45U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - src = *(in + 16 * 45 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 5U; - src = *(in + 16 * 46 + i); - tmp |= src << 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 9U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 23U; - src = *(in + 16 * 50 + i); - tmp |= src << 30U; - src = *(in + 16 * 51 + i); - tmp |= src << 37U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - src = *(in + 16 * 53 + i); - tmp |= src << 51U; - src = *(in + 16 * 54 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 1U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 15U; - src = *(in + 16 * 58 + i); - tmp |= src << 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 29U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 43U; - src = *(in + 16 * 62 + i); - tmp |= src << 50U; - src = *(in + 16 * 63 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out -= 96; - } -} -void pack_8bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 8U; - src = *(in + 16 * 2 + i); - tmp |= src << 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 24U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 40U; - src = *(in + 16 * 6 + i); - tmp |= src << 48U; - src = *(in + 16 * 7 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 8U; - src = *(in + 16 * 10 + i); - tmp |= src << 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 24U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 40U; - src = *(in + 16 * 14 + i); - tmp |= src << 48U; - src = *(in + 16 * 15 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 8U; - src = *(in + 16 * 18 + i); - tmp |= src << 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 24U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 40U; - src = *(in + 16 * 22 + i); - tmp |= src << 48U; - src = *(in + 16 * 23 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 8U; - src = *(in + 16 * 26 + i); - tmp |= src << 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 24U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 40U; - src = *(in + 16 * 30 + i); - tmp |= src << 48U; - src = *(in + 16 * 31 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 8U; - src = *(in + 16 * 34 + i); - tmp |= src << 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 24U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 40U; - src = *(in + 16 * 38 + i); - tmp |= src << 48U; - src = *(in + 16 * 39 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 8U; - src = *(in + 16 * 42 + i); - tmp |= src << 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 24U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 40U; - src = *(in + 16 * 46 + i); - tmp |= src << 48U; - src = *(in + 16 * 47 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 8U; - src = *(in + 16 * 50 + i); - tmp |= src << 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 24U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 40U; - src = *(in + 16 * 54 + i); - tmp |= src << 48U; - src = *(in + 16 * 55 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 8U; - src = *(in + 16 * 58 + i); - tmp |= src << 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 24U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 40U; - src = *(in + 16 * 62 + i); - tmp |= src << 48U; - src = *(in + 16 * 63 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out -= 112; - } -} -void pack_9bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 9U; - src = *(in + 16 * 2 + i); - tmp |= src << 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 27U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 45U; - src = *(in + 16 * 6 + i); - tmp |= src << 54U; - src = *(in + 16 * 7 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 1U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 17U; - src = *(in + 16 * 10 + i); - tmp |= src << 26U; - src = *(in + 16 * 11 + i); - tmp |= src << 35U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - src = *(in + 16 * 13 + i); - tmp |= src << 53U; - src = *(in + 16 * 14 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 7U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 25U; - src = *(in + 16 * 18 + i); - tmp |= src << 34U; - src = *(in + 16 * 19 + i); - tmp |= src << 43U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - src = *(in + 16 * 21 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 3U; - src = *(in + 16 * 22 + i); - tmp |= src << 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 15U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 33U; - src = *(in + 16 * 26 + i); - tmp |= src << 42U; - src = *(in + 16 * 27 + i); - tmp |= src << 51U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 5U; - src = *(in + 16 * 30 + i); - tmp |= src << 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 23U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 41U; - src = *(in + 16 * 34 + i); - tmp |= src << 50U; - src = *(in + 16 * 35 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 5U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 13U; - src = *(in + 16 * 38 + i); - tmp |= src << 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 31U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 49U; - src = *(in + 16 * 42 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 3U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 21U; - src = *(in + 16 * 46 + i); - tmp |= src << 30U; - src = *(in + 16 * 47 + i); - tmp |= src << 39U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 7U; - src = *(in + 16 * 50 + i); - tmp |= src << 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 11U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 29U; - src = *(in + 16 * 54 + i); - tmp |= src << 38U; - src = *(in + 16 * 55 + i); - tmp |= src << 47U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 1U; - src = *(in + 16 * 58 + i); - tmp |= src << 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 19U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 37U; - src = *(in + 16 * 62 + i); - tmp |= src << 46U; - src = *(in + 16 * 63 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out -= 128; - } -} -void pack_10bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 10U; - src = *(in + 16 * 2 + i); - tmp |= src << 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 30U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - src = *(in + 16 * 5 + i); - tmp |= src << 50U; - src = *(in + 16 * 6 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 6U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 26U; - src = *(in + 16 * 10 + i); - tmp |= src << 36U; - src = *(in + 16 * 11 + i); - tmp |= src << 46U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 2U; - src = *(in + 16 * 14 + i); - tmp |= src << 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 22U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 42U; - src = *(in + 16 * 18 + i); - tmp |= src << 52U; - src = *(in + 16 * 19 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 2U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 18U; - src = *(in + 16 * 22 + i); - tmp |= src << 28U; - src = *(in + 16 * 23 + i); - tmp |= src << 38U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 6U; - src = *(in + 16 * 26 + i); - tmp |= src << 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 14U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 34U; - src = *(in + 16 * 30 + i); - tmp |= src << 44U; - src = *(in + 16 * 31 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 10U; - src = *(in + 16 * 34 + i); - tmp |= src << 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 30U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - src = *(in + 16 * 37 + i); - tmp |= src << 50U; - src = *(in + 16 * 38 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 6U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 26U; - src = *(in + 16 * 42 + i); - tmp |= src << 36U; - src = *(in + 16 * 43 + i); - tmp |= src << 46U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 2U; - src = *(in + 16 * 46 + i); - tmp |= src << 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 22U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 42U; - src = *(in + 16 * 50 + i); - tmp |= src << 52U; - src = *(in + 16 * 51 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 2U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 18U; - src = *(in + 16 * 54 + i); - tmp |= src << 28U; - src = *(in + 16 * 55 + i); - tmp |= src << 38U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 6U; - src = *(in + 16 * 58 + i); - tmp |= src << 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 14U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 34U; - src = *(in + 16 * 62 + i); - tmp |= src << 44U; - src = *(in + 16 * 63 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out -= 144; - } -} -void pack_11bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 11U; - src = *(in + 16 * 2 + i); - tmp |= src << 22U; - src = *(in + 16 * 3 + i); - tmp |= src << 33U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - src = *(in + 16 * 5 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 9U; - src = *(in + 16 * 6 + i); - tmp |= src << 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 13U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 35U; - src = *(in + 16 * 10 + i); - tmp |= src << 46U; - src = *(in + 16 * 11 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 7U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 15U; - src = *(in + 16 * 14 + i); - tmp |= src << 26U; - src = *(in + 16 * 15 + i); - tmp |= src << 37U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 5U; - src = *(in + 16 * 18 + i); - tmp |= src << 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 17U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 39U; - src = *(in + 16 * 22 + i); - tmp |= src << 50U; - src = *(in + 16 * 23 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 3U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 19U; - src = *(in + 16 * 26 + i); - tmp |= src << 30U; - src = *(in + 16 * 27 + i); - tmp |= src << 41U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - src = *(in + 16 * 29 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 1U; - src = *(in + 16 * 30 + i); - tmp |= src << 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 21U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 43U; - src = *(in + 16 * 34 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 1U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 23U; - src = *(in + 16 * 38 + i); - tmp |= src << 34U; - src = *(in + 16 * 39 + i); - tmp |= src << 45U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 3U; - src = *(in + 16 * 42 + i); - tmp |= src << 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 25U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 47U; - src = *(in + 16 * 46 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 5U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 27U; - src = *(in + 16 * 50 + i); - tmp |= src << 38U; - src = *(in + 16 * 51 + i); - tmp |= src << 49U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 7U; - src = *(in + 16 * 54 + i); - tmp |= src << 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 29U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 51U; - src = *(in + 16 * 58 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 9U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 31U; - src = *(in + 16 * 62 + i); - tmp |= src << 42U; - src = *(in + 16 * 63 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out -= 160; - } -} -void pack_12bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 12U; - src = *(in + 16 * 2 + i); - tmp |= src << 24U; - src = *(in + 16 * 3 + i); - tmp |= src << 36U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - src = *(in + 16 * 5 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 4U; - src = *(in + 16 * 6 + i); - tmp |= src << 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 20U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 44U; - src = *(in + 16 * 10 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 4U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 28U; - src = *(in + 16 * 14 + i); - tmp |= src << 40U; - src = *(in + 16 * 15 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 12U; - src = *(in + 16 * 18 + i); - tmp |= src << 24U; - src = *(in + 16 * 19 + i); - tmp |= src << 36U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - src = *(in + 16 * 21 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 4U; - src = *(in + 16 * 22 + i); - tmp |= src << 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 20U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 44U; - src = *(in + 16 * 26 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 4U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 28U; - src = *(in + 16 * 30 + i); - tmp |= src << 40U; - src = *(in + 16 * 31 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 12U; - src = *(in + 16 * 34 + i); - tmp |= src << 24U; - src = *(in + 16 * 35 + i); - tmp |= src << 36U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - src = *(in + 16 * 37 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 4U; - src = *(in + 16 * 38 + i); - tmp |= src << 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 20U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 44U; - src = *(in + 16 * 42 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 4U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 28U; - src = *(in + 16 * 46 + i); - tmp |= src << 40U; - src = *(in + 16 * 47 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 12U; - src = *(in + 16 * 50 + i); - tmp |= src << 24U; - src = *(in + 16 * 51 + i); - tmp |= src << 36U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - src = *(in + 16 * 53 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 4U; - src = *(in + 16 * 54 + i); - tmp |= src << 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 20U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 44U; - src = *(in + 16 * 58 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 4U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 28U; - src = *(in + 16 * 62 + i); - tmp |= src << 40U; - src = *(in + 16 * 63 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out -= 176; - } -} -void pack_13bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 13U; - src = *(in + 16 * 2 + i); - tmp |= src << 26U; - src = *(in + 16 * 3 + i); - tmp |= src << 39U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 1U; - src = *(in + 16 * 6 + i); - tmp |= src << 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 27U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 11U; - src = *(in + 16 * 10 + i); - tmp |= src << 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 15U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 41U; - src = *(in + 16 * 14 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 3U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 29U; - src = *(in + 16 * 18 + i); - tmp |= src << 42U; - src = *(in + 16 * 19 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 9U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 17U; - src = *(in + 16 * 22 + i); - tmp |= src << 30U; - src = *(in + 16 * 23 + i); - tmp |= src << 43U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 5U; - src = *(in + 16 * 26 + i); - tmp |= src << 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 31U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - src = *(in + 16 * 29 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 7U; - src = *(in + 16 * 30 + i); - tmp |= src << 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 19U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 45U; - src = *(in + 16 * 34 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 7U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 33U; - src = *(in + 16 * 38 + i); - tmp |= src << 46U; - src = *(in + 16 * 39 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 5U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 21U; - src = *(in + 16 * 42 + i); - tmp |= src << 34U; - src = *(in + 16 * 43 + i); - tmp |= src << 47U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 9U; - src = *(in + 16 * 46 + i); - tmp |= src << 22U; - src = *(in + 16 * 47 + i); - tmp |= src << 35U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 3U; - src = *(in + 16 * 50 + i); - tmp |= src << 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 23U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - src = *(in + 16 * 53 + i); - tmp |= src << 49U; - src = *(in + 16 * 54 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 11U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 37U; - src = *(in + 16 * 58 + i); - tmp |= src << 50U; - src = *(in + 16 * 59 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 1U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 25U; - src = *(in + 16 * 62 + i); - tmp |= src << 38U; - src = *(in + 16 * 63 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out -= 192; - } -} -void pack_14bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 14U; - src = *(in + 16 * 2 + i); - tmp |= src << 28U; - src = *(in + 16 * 3 + i); - tmp |= src << 42U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 6U; - src = *(in + 16 * 6 + i); - tmp |= src << 20U; - src = *(in + 16 * 7 + i); - tmp |= src << 34U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 2U; - src = *(in + 16 * 10 + i); - tmp |= src << 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 26U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - src = *(in + 16 * 13 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 10U; - src = *(in + 16 * 14 + i); - tmp |= src << 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 18U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 46U; - src = *(in + 16 * 18 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 10U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 38U; - src = *(in + 16 * 22 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 2U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 30U; - src = *(in + 16 * 26 + i); - tmp |= src << 44U; - src = *(in + 16 * 27 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 6U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 22U; - src = *(in + 16 * 30 + i); - tmp |= src << 36U; - src = *(in + 16 * 31 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 14U; - src = *(in + 16 * 34 + i); - tmp |= src << 28U; - src = *(in + 16 * 35 + i); - tmp |= src << 42U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 6U; - src = *(in + 16 * 38 + i); - tmp |= src << 20U; - src = *(in + 16 * 39 + i); - tmp |= src << 34U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 2U; - src = *(in + 16 * 42 + i); - tmp |= src << 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 26U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - src = *(in + 16 * 45 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 10U; - src = *(in + 16 * 46 + i); - tmp |= src << 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 18U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 46U; - src = *(in + 16 * 50 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 10U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 38U; - src = *(in + 16 * 54 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 2U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 30U; - src = *(in + 16 * 58 + i); - tmp |= src << 44U; - src = *(in + 16 * 59 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 6U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 22U; - src = *(in + 16 * 62 + i); - tmp |= src << 36U; - src = *(in + 16 * 63 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out -= 208; - } -} -void pack_15bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 15U; - src = *(in + 16 * 2 + i); - tmp |= src << 30U; - src = *(in + 16 * 3 + i); - tmp |= src << 45U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 11U; - src = *(in + 16 * 6 + i); - tmp |= src << 26U; - src = *(in + 16 * 7 + i); - tmp |= src << 41U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 7U; - src = *(in + 16 * 10 + i); - tmp |= src << 22U; - src = *(in + 16 * 11 + i); - tmp |= src << 37U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 3U; - src = *(in + 16 * 14 + i); - tmp |= src << 18U; - src = *(in + 16 * 15 + i); - tmp |= src << 33U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 1U; - src = *(in + 16 * 18 + i); - tmp |= src << 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 29U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - src = *(in + 16 * 21 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 5U; - src = *(in + 16 * 22 + i); - tmp |= src << 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 25U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 9U; - src = *(in + 16 * 26 + i); - tmp |= src << 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 21U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - src = *(in + 16 * 29 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 13U; - src = *(in + 16 * 30 + i); - tmp |= src << 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 17U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 47U; - src = *(in + 16 * 34 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 13U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 43U; - src = *(in + 16 * 38 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 9U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 39U; - src = *(in + 16 * 42 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 5U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 35U; - src = *(in + 16 * 46 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 1U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 31U; - src = *(in + 16 * 50 + i); - tmp |= src << 46U; - src = *(in + 16 * 51 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 3U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 27U; - src = *(in + 16 * 54 + i); - tmp |= src << 42U; - src = *(in + 16 * 55 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 7U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 23U; - src = *(in + 16 * 58 + i); - tmp |= src << 38U; - src = *(in + 16 * 59 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 11U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 19U; - src = *(in + 16 * 62 + i); - tmp |= src << 34U; - src = *(in + 16 * 63 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out -= 224; - } -} -void pack_16bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 16U; - src = *(in + 16 * 2 + i); - tmp |= src << 32U; - src = *(in + 16 * 3 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - src = *(in + 16 * 5 + i); - tmp |= src << 16U; - src = *(in + 16 * 6 + i); - tmp |= src << 32U; - src = *(in + 16 * 7 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 16U; - src = *(in + 16 * 10 + i); - tmp |= src << 32U; - src = *(in + 16 * 11 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - src = *(in + 16 * 13 + i); - tmp |= src << 16U; - src = *(in + 16 * 14 + i); - tmp |= src << 32U; - src = *(in + 16 * 15 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 16U; - src = *(in + 16 * 18 + i); - tmp |= src << 32U; - src = *(in + 16 * 19 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - src = *(in + 16 * 21 + i); - tmp |= src << 16U; - src = *(in + 16 * 22 + i); - tmp |= src << 32U; - src = *(in + 16 * 23 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 16U; - src = *(in + 16 * 26 + i); - tmp |= src << 32U; - src = *(in + 16 * 27 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - src = *(in + 16 * 29 + i); - tmp |= src << 16U; - src = *(in + 16 * 30 + i); - tmp |= src << 32U; - src = *(in + 16 * 31 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 16U; - src = *(in + 16 * 34 + i); - tmp |= src << 32U; - src = *(in + 16 * 35 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - src = *(in + 16 * 37 + i); - tmp |= src << 16U; - src = *(in + 16 * 38 + i); - tmp |= src << 32U; - src = *(in + 16 * 39 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 16U; - src = *(in + 16 * 42 + i); - tmp |= src << 32U; - src = *(in + 16 * 43 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - src = *(in + 16 * 45 + i); - tmp |= src << 16U; - src = *(in + 16 * 46 + i); - tmp |= src << 32U; - src = *(in + 16 * 47 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 16U; - src = *(in + 16 * 50 + i); - tmp |= src << 32U; - src = *(in + 16 * 51 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - src = *(in + 16 * 53 + i); - tmp |= src << 16U; - src = *(in + 16 * 54 + i); - tmp |= src << 32U; - src = *(in + 16 * 55 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 16U; - src = *(in + 16 * 58 + i); - tmp |= src << 32U; - src = *(in + 16 * 59 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - src = *(in + 16 * 61 + i); - tmp |= src << 16U; - src = *(in + 16 * 62 + i); - tmp |= src << 32U; - src = *(in + 16 * 63 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out -= 240; - } -} -void pack_17bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 17U; - src = *(in + 16 * 2 + i); - tmp |= src << 34U; - src = *(in + 16 * 3 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 13U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 21U; - src = *(in + 16 * 6 + i); - tmp |= src << 38U; - src = *(in + 16 * 7 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 9U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 25U; - src = *(in + 16 * 10 + i); - tmp |= src << 42U; - src = *(in + 16 * 11 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 5U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 29U; - src = *(in + 16 * 14 + i); - tmp |= src << 46U; - src = *(in + 16 * 15 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 1U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 33U; - src = *(in + 16 * 18 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 3U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 37U; - src = *(in + 16 * 22 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 7U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 41U; - src = *(in + 16 * 26 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 11U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 45U; - src = *(in + 16 * 30 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 15U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 15U; - src = *(in + 16 * 34 + i); - tmp |= src << 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 19U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 11U; - src = *(in + 16 * 38 + i); - tmp |= src << 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 23U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 7U; - src = *(in + 16 * 42 + i); - tmp |= src << 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 27U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 3U; - src = *(in + 16 * 46 + i); - tmp |= src << 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 31U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 1U; - src = *(in + 16 * 50 + i); - tmp |= src << 18U; - src = *(in + 16 * 51 + i); - tmp |= src << 35U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 5U; - src = *(in + 16 * 54 + i); - tmp |= src << 22U; - src = *(in + 16 * 55 + i); - tmp |= src << 39U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 9U; - src = *(in + 16 * 58 + i); - tmp |= src << 26U; - src = *(in + 16 * 59 + i); - tmp |= src << 43U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 13U; - src = *(in + 16 * 62 + i); - tmp |= src << 30U; - src = *(in + 16 * 63 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out -= 256; - } -} -void pack_18bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 18U; - src = *(in + 16 * 2 + i); - tmp |= src << 36U; - src = *(in + 16 * 3 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 10U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 26U; - src = *(in + 16 * 6 + i); - tmp |= src << 44U; - src = *(in + 16 * 7 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 2U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 34U; - src = *(in + 16 * 10 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 6U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 42U; - src = *(in + 16 * 14 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 14U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 14U; - src = *(in + 16 * 18 + i); - tmp |= src << 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 22U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 6U; - src = *(in + 16 * 22 + i); - tmp |= src << 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 30U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 2U; - src = *(in + 16 * 26 + i); - tmp |= src << 20U; - src = *(in + 16 * 27 + i); - tmp |= src << 38U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 10U; - src = *(in + 16 * 30 + i); - tmp |= src << 28U; - src = *(in + 16 * 31 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 18U; - src = *(in + 16 * 34 + i); - tmp |= src << 36U; - src = *(in + 16 * 35 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 10U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 26U; - src = *(in + 16 * 38 + i); - tmp |= src << 44U; - src = *(in + 16 * 39 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 2U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 34U; - src = *(in + 16 * 42 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 6U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 42U; - src = *(in + 16 * 46 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 14U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 14U; - src = *(in + 16 * 50 + i); - tmp |= src << 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 22U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 6U; - src = *(in + 16 * 54 + i); - tmp |= src << 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 30U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 2U; - src = *(in + 16 * 58 + i); - tmp |= src << 20U; - src = *(in + 16 * 59 + i); - tmp |= src << 38U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 10U; - src = *(in + 16 * 62 + i); - tmp |= src << 28U; - src = *(in + 16 * 63 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out -= 272; - } -} -void pack_19bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 19U; - src = *(in + 16 * 2 + i); - tmp |= src << 38U; - src = *(in + 16 * 3 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 7U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 31U; - src = *(in + 16 * 6 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 5U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 43U; - src = *(in + 16 * 10 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 17U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 9U; - src = *(in + 16 * 14 + i); - tmp |= src << 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 29U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 3U; - src = *(in + 16 * 18 + i); - tmp |= src << 22U; - src = *(in + 16 * 19 + i); - tmp |= src << 41U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 15U; - src = *(in + 16 * 22 + i); - tmp |= src << 34U; - src = *(in + 16 * 23 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 11U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 27U; - src = *(in + 16 * 26 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 1U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 39U; - src = *(in + 16 * 30 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 13U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 13U; - src = *(in + 16 * 34 + i); - tmp |= src << 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 25U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 1U; - src = *(in + 16 * 38 + i); - tmp |= src << 18U; - src = *(in + 16 * 39 + i); - tmp |= src << 37U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 11U; - src = *(in + 16 * 42 + i); - tmp |= src << 30U; - src = *(in + 16 * 43 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 15U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 23U; - src = *(in + 16 * 46 + i); - tmp |= src << 42U; - src = *(in + 16 * 47 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 3U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 35U; - src = *(in + 16 * 50 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 9U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 17U; - src = *(in + 16 * 54 + i); - tmp |= src << 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 21U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 5U; - src = *(in + 16 * 58 + i); - tmp |= src << 14U; - src = *(in + 16 * 59 + i); - tmp |= src << 33U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 7U; - src = *(in + 16 * 62 + i); - tmp |= src << 26U; - src = *(in + 16 * 63 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out -= 288; - } -} -void pack_20bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 20U; - src = *(in + 16 * 2 + i); - tmp |= src << 40U; - src = *(in + 16 * 3 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 4U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 36U; - src = *(in + 16 * 6 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 12U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 12U; - src = *(in + 16 * 10 + i); - tmp |= src << 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 28U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 4U; - src = *(in + 16 * 14 + i); - tmp |= src << 24U; - src = *(in + 16 * 15 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 20U; - src = *(in + 16 * 18 + i); - tmp |= src << 40U; - src = *(in + 16 * 19 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 4U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 36U; - src = *(in + 16 * 22 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 12U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 12U; - src = *(in + 16 * 26 + i); - tmp |= src << 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 28U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 4U; - src = *(in + 16 * 30 + i); - tmp |= src << 24U; - src = *(in + 16 * 31 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 20U; - src = *(in + 16 * 34 + i); - tmp |= src << 40U; - src = *(in + 16 * 35 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 4U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 36U; - src = *(in + 16 * 38 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 12U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 12U; - src = *(in + 16 * 42 + i); - tmp |= src << 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 28U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 4U; - src = *(in + 16 * 46 + i); - tmp |= src << 24U; - src = *(in + 16 * 47 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 20U; - src = *(in + 16 * 50 + i); - tmp |= src << 40U; - src = *(in + 16 * 51 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 4U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 36U; - src = *(in + 16 * 54 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 12U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 12U; - src = *(in + 16 * 58 + i); - tmp |= src << 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 28U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 4U; - src = *(in + 16 * 62 + i); - tmp |= src << 24U; - src = *(in + 16 * 63 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out -= 304; - } -} -void pack_21bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 21U; - src = *(in + 16 * 2 + i); - tmp |= src << 42U; - src = *(in + 16 * 3 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 1U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 41U; - src = *(in + 16 * 6 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 19U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 3U; - src = *(in + 16 * 10 + i); - tmp |= src << 18U; - src = *(in + 16 * 11 + i); - tmp |= src << 39U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 17U; - src = *(in + 16 * 14 + i); - tmp |= src << 38U; - src = *(in + 16 * 15 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 5U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 37U; - src = *(in + 16 * 18 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 15U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 7U; - src = *(in + 16 * 22 + i); - tmp |= src << 14U; - src = *(in + 16 * 23 + i); - tmp |= src << 35U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 13U; - src = *(in + 16 * 26 + i); - tmp |= src << 34U; - src = *(in + 16 * 27 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 9U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 33U; - src = *(in + 16 * 30 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 11U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 11U; - src = *(in + 16 * 34 + i); - tmp |= src << 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 31U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 9U; - src = *(in + 16 * 38 + i); - tmp |= src << 30U; - src = *(in + 16 * 39 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 13U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 29U; - src = *(in + 16 * 42 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 7U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 15U; - src = *(in + 16 * 46 + i); - tmp |= src << 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 27U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 5U; - src = *(in + 16 * 50 + i); - tmp |= src << 26U; - src = *(in + 16 * 51 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 17U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 25U; - src = *(in + 16 * 54 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 3U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 19U; - src = *(in + 16 * 58 + i); - tmp |= src << 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 23U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 1U; - src = *(in + 16 * 62 + i); - tmp |= src << 22U; - src = *(in + 16 * 63 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out -= 320; - } -} -void pack_22bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 22U; - src = *(in + 16 * 2 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 2U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 18U; - src = *(in + 16 * 6 + i); - tmp |= src << 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 26U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 6U; - src = *(in + 16 * 10 + i); - tmp |= src << 28U; - src = *(in + 16 * 11 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 14U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 30U; - src = *(in + 16 * 14 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 10U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 10U; - src = *(in + 16 * 18 + i); - tmp |= src << 12U; - src = *(in + 16 * 19 + i); - tmp |= src << 34U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 14U; - src = *(in + 16 * 22 + i); - tmp |= src << 36U; - src = *(in + 16 * 23 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 6U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 38U; - src = *(in + 16 * 26 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 18U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 2U; - src = *(in + 16 * 30 + i); - tmp |= src << 20U; - src = *(in + 16 * 31 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 22U; - src = *(in + 16 * 34 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 2U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 18U; - src = *(in + 16 * 38 + i); - tmp |= src << 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 26U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 6U; - src = *(in + 16 * 42 + i); - tmp |= src << 28U; - src = *(in + 16 * 43 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 14U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 30U; - src = *(in + 16 * 46 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 10U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 10U; - src = *(in + 16 * 50 + i); - tmp |= src << 12U; - src = *(in + 16 * 51 + i); - tmp |= src << 34U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 14U; - src = *(in + 16 * 54 + i); - tmp |= src << 36U; - src = *(in + 16 * 55 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 6U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 38U; - src = *(in + 16 * 58 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 18U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 2U; - src = *(in + 16 * 62 + i); - tmp |= src << 20U; - src = *(in + 16 * 63 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out -= 336; - } -} -void pack_23bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 23U; - src = *(in + 16 * 2 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 5U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 13U; - src = *(in + 16 * 6 + i); - tmp |= src << 10U; - src = *(in + 16 * 7 + i); - tmp |= src << 33U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 15U; - src = *(in + 16 * 10 + i); - tmp |= src << 38U; - src = *(in + 16 * 11 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 3U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 21U; - src = *(in + 16 * 14 + i); - tmp |= src << 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 25U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 7U; - src = *(in + 16 * 18 + i); - tmp |= src << 30U; - src = *(in + 16 * 19 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 11U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 35U; - src = *(in + 16 * 22 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 17U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 1U; - src = *(in + 16 * 26 + i); - tmp |= src << 22U; - src = *(in + 16 * 27 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 19U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 27U; - src = *(in + 16 * 30 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 9U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 9U; - src = *(in + 16 * 34 + i); - tmp |= src << 14U; - src = *(in + 16 * 35 + i); - tmp |= src << 37U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 19U; - src = *(in + 16 * 38 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 1U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 17U; - src = *(in + 16 * 42 + i); - tmp |= src << 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 29U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 11U; - src = *(in + 16 * 46 + i); - tmp |= src << 34U; - src = *(in + 16 * 47 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 7U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 39U; - src = *(in + 16 * 50 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 21U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 3U; - src = *(in + 16 * 54 + i); - tmp |= src << 26U; - src = *(in + 16 * 55 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 15U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 31U; - src = *(in + 16 * 58 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 13U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 5U; - src = *(in + 16 * 62 + i); - tmp |= src << 18U; - src = *(in + 16 * 63 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out -= 352; - } -} -void pack_24bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 24U; - src = *(in + 16 * 2 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 8U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 8U; - src = *(in + 16 * 6 + i); - tmp |= src << 16U; - src = *(in + 16 * 7 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 24U; - src = *(in + 16 * 10 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 8U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 8U; - src = *(in + 16 * 14 + i); - tmp |= src << 16U; - src = *(in + 16 * 15 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 24U; - src = *(in + 16 * 18 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 8U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 8U; - src = *(in + 16 * 22 + i); - tmp |= src << 16U; - src = *(in + 16 * 23 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 24U; - src = *(in + 16 * 26 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 8U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 8U; - src = *(in + 16 * 30 + i); - tmp |= src << 16U; - src = *(in + 16 * 31 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 24U; - src = *(in + 16 * 34 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 8U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 8U; - src = *(in + 16 * 38 + i); - tmp |= src << 16U; - src = *(in + 16 * 39 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 24U; - src = *(in + 16 * 42 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 8U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 8U; - src = *(in + 16 * 46 + i); - tmp |= src << 16U; - src = *(in + 16 * 47 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 24U; - src = *(in + 16 * 50 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 8U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 8U; - src = *(in + 16 * 54 + i); - tmp |= src << 16U; - src = *(in + 16 * 55 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 24U; - src = *(in + 16 * 58 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 8U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 8U; - src = *(in + 16 * 62 + i); - tmp |= src << 16U; - src = *(in + 16 * 63 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out -= 368; - } -} -void pack_25bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 25U; - src = *(in + 16 * 2 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 11U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 3U; - src = *(in + 16 * 6 + i); - tmp |= src << 22U; - src = *(in + 16 * 7 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 17U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 33U; - src = *(in + 16 * 10 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 19U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 5U; - src = *(in + 16 * 14 + i); - tmp |= src << 30U; - src = *(in + 16 * 15 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 9U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 23U; - src = *(in + 16 * 18 + i); - tmp |= src << 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 27U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 13U; - src = *(in + 16 * 22 + i); - tmp |= src << 38U; - src = *(in + 16 * 23 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 1U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 15U; - src = *(in + 16 * 26 + i); - tmp |= src << 10U; - src = *(in + 16 * 27 + i); - tmp |= src << 35U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 21U; - src = *(in + 16 * 30 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 7U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 7U; - src = *(in + 16 * 34 + i); - tmp |= src << 18U; - src = *(in + 16 * 35 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 21U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 29U; - src = *(in + 16 * 38 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 15U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 1U; - src = *(in + 16 * 42 + i); - tmp |= src << 26U; - src = *(in + 16 * 43 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 13U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 37U; - src = *(in + 16 * 46 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 23U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 9U; - src = *(in + 16 * 50 + i); - tmp |= src << 34U; - src = *(in + 16 * 51 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 5U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 19U; - src = *(in + 16 * 54 + i); - tmp |= src << 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 31U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 17U; - src = *(in + 16 * 58 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 3U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 11U; - src = *(in + 16 * 62 + i); - tmp |= src << 14U; - src = *(in + 16 * 63 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out -= 384; - } -} -void pack_26bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 26U; - src = *(in + 16 * 2 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 14U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 2U; - src = *(in + 16 * 6 + i); - tmp |= src << 28U; - src = *(in + 16 * 7 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 10U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 22U; - src = *(in + 16 * 10 + i); - tmp |= src << 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 30U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 18U; - src = *(in + 16 * 14 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 6U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 6U; - src = *(in + 16 * 18 + i); - tmp |= src << 20U; - src = *(in + 16 * 19 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 18U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 34U; - src = *(in + 16 * 22 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 22U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 10U; - src = *(in + 16 * 26 + i); - tmp |= src << 36U; - src = *(in + 16 * 27 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 2U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 14U; - src = *(in + 16 * 30 + i); - tmp |= src << 12U; - src = *(in + 16 * 31 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 26U; - src = *(in + 16 * 34 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 14U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 2U; - src = *(in + 16 * 38 + i); - tmp |= src << 28U; - src = *(in + 16 * 39 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 10U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 22U; - src = *(in + 16 * 42 + i); - tmp |= src << 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 30U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 18U; - src = *(in + 16 * 46 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 6U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 6U; - src = *(in + 16 * 50 + i); - tmp |= src << 20U; - src = *(in + 16 * 51 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 18U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 34U; - src = *(in + 16 * 54 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 22U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 10U; - src = *(in + 16 * 58 + i); - tmp |= src << 36U; - src = *(in + 16 * 59 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 2U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 14U; - src = *(in + 16 * 62 + i); - tmp |= src << 12U; - src = *(in + 16 * 63 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out -= 400; - } -} -void pack_27bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 27U; - src = *(in + 16 * 2 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 17U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 7U; - src = *(in + 16 * 6 + i); - tmp |= src << 34U; - src = *(in + 16 * 7 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 3U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 13U; - src = *(in + 16 * 10 + i); - tmp |= src << 14U; - src = *(in + 16 * 11 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 23U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 31U; - src = *(in + 16 * 14 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 21U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 11U; - src = *(in + 16 * 18 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 1U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 9U; - src = *(in + 16 * 22 + i); - tmp |= src << 18U; - src = *(in + 16 * 23 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 19U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 35U; - src = *(in + 16 * 26 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 25U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 15U; - src = *(in + 16 * 30 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 5U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 5U; - src = *(in + 16 * 34 + i); - tmp |= src << 22U; - src = *(in + 16 * 35 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 15U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 25U; - src = *(in + 16 * 38 + i); - tmp |= src << 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 29U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 19U; - src = *(in + 16 * 42 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 9U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 1U; - src = *(in + 16 * 46 + i); - tmp |= src << 26U; - src = *(in + 16 * 47 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 11U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 21U; - src = *(in + 16 * 50 + i); - tmp |= src << 6U; - src = *(in + 16 * 51 + i); - tmp |= src << 33U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 23U; - src = *(in + 16 * 54 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 13U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 3U; - src = *(in + 16 * 58 + i); - tmp |= src << 30U; - src = *(in + 16 * 59 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 7U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 17U; - src = *(in + 16 * 62 + i); - tmp |= src << 10U; - src = *(in + 16 * 63 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out -= 416; - } -} -void pack_28bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 28U; - src = *(in + 16 * 2 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 20U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 12U; - src = *(in + 16 * 6 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 4U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 4U; - src = *(in + 16 * 10 + i); - tmp |= src << 24U; - src = *(in + 16 * 11 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 12U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 20U; - src = *(in + 16 * 14 + i); - tmp |= src << 8U; - src = *(in + 16 * 15 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 28U; - src = *(in + 16 * 18 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 20U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 12U; - src = *(in + 16 * 22 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 4U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 4U; - src = *(in + 16 * 26 + i); - tmp |= src << 24U; - src = *(in + 16 * 27 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 12U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 20U; - src = *(in + 16 * 30 + i); - tmp |= src << 8U; - src = *(in + 16 * 31 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 28U; - src = *(in + 16 * 34 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 20U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 12U; - src = *(in + 16 * 38 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 4U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 4U; - src = *(in + 16 * 42 + i); - tmp |= src << 24U; - src = *(in + 16 * 43 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 12U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 20U; - src = *(in + 16 * 46 + i); - tmp |= src << 8U; - src = *(in + 16 * 47 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 28U; - src = *(in + 16 * 50 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 20U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 12U; - src = *(in + 16 * 54 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 4U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 4U; - src = *(in + 16 * 58 + i); - tmp |= src << 24U; - src = *(in + 16 * 59 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 12U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 20U; - src = *(in + 16 * 62 + i); - tmp |= src << 8U; - src = *(in + 16 * 63 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out -= 432; - } -} -void pack_29bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 29U; - src = *(in + 16 * 2 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 23U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 17U; - src = *(in + 16 * 6 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 11U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 5U; - src = *(in + 16 * 10 + i); - tmp |= src << 34U; - src = *(in + 16 * 11 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 1U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 7U; - src = *(in + 16 * 14 + i); - tmp |= src << 22U; - src = *(in + 16 * 15 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 13U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 19U; - src = *(in + 16 * 18 + i); - tmp |= src << 10U; - src = *(in + 16 * 19 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 25U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 33U; - src = *(in + 16 * 22 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 27U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 21U; - src = *(in + 16 * 26 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 15U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 9U; - src = *(in + 16 * 30 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 3U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 3U; - src = *(in + 16 * 34 + i); - tmp |= src << 26U; - src = *(in + 16 * 35 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 9U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 15U; - src = *(in + 16 * 38 + i); - tmp |= src << 14U; - src = *(in + 16 * 39 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 21U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 27U; - src = *(in + 16 * 42 + i); - tmp |= src << 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 31U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 25U; - src = *(in + 16 * 46 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 19U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 13U; - src = *(in + 16 * 50 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 7U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 1U; - src = *(in + 16 * 54 + i); - tmp |= src << 30U; - src = *(in + 16 * 55 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 5U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 11U; - src = *(in + 16 * 58 + i); - tmp |= src << 18U; - src = *(in + 16 * 59 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 17U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 23U; - src = *(in + 16 * 62 + i); - tmp |= src << 6U; - src = *(in + 16 * 63 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out -= 448; - } -} -void pack_30bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 30U; - src = *(in + 16 * 2 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 26U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 22U; - src = *(in + 16 * 6 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 18U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 14U; - src = *(in + 16 * 10 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 10U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 6U; - src = *(in + 16 * 14 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 2U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 2U; - src = *(in + 16 * 18 + i); - tmp |= src << 28U; - src = *(in + 16 * 19 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 6U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 10U; - src = *(in + 16 * 22 + i); - tmp |= src << 20U; - src = *(in + 16 * 23 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 14U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 18U; - src = *(in + 16 * 26 + i); - tmp |= src << 12U; - src = *(in + 16 * 27 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 22U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 26U; - src = *(in + 16 * 30 + i); - tmp |= src << 4U; - src = *(in + 16 * 31 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 30U; - src = *(in + 16 * 34 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 26U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 22U; - src = *(in + 16 * 38 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 18U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 14U; - src = *(in + 16 * 42 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 10U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 6U; - src = *(in + 16 * 46 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 2U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 2U; - src = *(in + 16 * 50 + i); - tmp |= src << 28U; - src = *(in + 16 * 51 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 6U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 10U; - src = *(in + 16 * 54 + i); - tmp |= src << 20U; - src = *(in + 16 * 55 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 14U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 18U; - src = *(in + 16 * 58 + i); - tmp |= src << 12U; - src = *(in + 16 * 59 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 22U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 26U; - src = *(in + 16 * 62 + i); - tmp |= src << 4U; - src = *(in + 16 * 63 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out -= 464; - } -} -void pack_31bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 31U; - src = *(in + 16 * 2 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 29U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 27U; - src = *(in + 16 * 6 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 25U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 23U; - src = *(in + 16 * 10 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 21U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 19U; - src = *(in + 16 * 14 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 17U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 15U; - src = *(in + 16 * 18 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 13U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 11U; - src = *(in + 16 * 22 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 9U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 7U; - src = *(in + 16 * 26 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 5U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 3U; - src = *(in + 16 * 30 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 1U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 1U; - src = *(in + 16 * 34 + i); - tmp |= src << 30U; - src = *(in + 16 * 35 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 3U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 5U; - src = *(in + 16 * 38 + i); - tmp |= src << 26U; - src = *(in + 16 * 39 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 7U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 9U; - src = *(in + 16 * 42 + i); - tmp |= src << 22U; - src = *(in + 16 * 43 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 11U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 13U; - src = *(in + 16 * 46 + i); - tmp |= src << 18U; - src = *(in + 16 * 47 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 15U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 17U; - src = *(in + 16 * 50 + i); - tmp |= src << 14U; - src = *(in + 16 * 51 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 19U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 21U; - src = *(in + 16 * 54 + i); - tmp |= src << 10U; - src = *(in + 16 * 55 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 23U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 25U; - src = *(in + 16 * 58 + i); - tmp |= src << 6U; - src = *(in + 16 * 59 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 27U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 29U; - src = *(in + 16 * 62 + i); - tmp |= src << 2U; - src = *(in + 16 * 63 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out -= 480; - } -} -void pack_32bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src; - src = *(in + 16 * 3 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - src = *(in + 16 * 5 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src; - src = *(in + 16 * 7 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src; - src = *(in + 16 * 11 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - src = *(in + 16 * 13 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src; - src = *(in + 16 * 15 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src; - src = *(in + 16 * 19 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - src = *(in + 16 * 21 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src; - src = *(in + 16 * 23 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src; - src = *(in + 16 * 27 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - src = *(in + 16 * 29 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src; - src = *(in + 16 * 31 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src; - src = *(in + 16 * 35 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - src = *(in + 16 * 37 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src; - src = *(in + 16 * 39 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src; - src = *(in + 16 * 43 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - src = *(in + 16 * 45 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src; - src = *(in + 16 * 47 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src; - src = *(in + 16 * 51 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - src = *(in + 16 * 53 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src; - src = *(in + 16 * 55 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src; - src = *(in + 16 * 59 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - src = *(in + 16 * 61 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src; - src = *(in + 16 * 63 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out -= 496; - } -} -void pack_33bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 31U; - src = *(in + 16 * 2 + i); - tmp |= src << 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 29U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 27U; - src = *(in + 16 * 6 + i); - tmp |= src << 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 25U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 23U; - src = *(in + 16 * 10 + i); - tmp |= src << 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 21U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 19U; - src = *(in + 16 * 14 + i); - tmp |= src << 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 17U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 15U; - src = *(in + 16 * 18 + i); - tmp |= src << 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 13U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 11U; - src = *(in + 16 * 22 + i); - tmp |= src << 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 9U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 7U; - src = *(in + 16 * 26 + i); - tmp |= src << 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 5U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 3U; - src = *(in + 16 * 30 + i); - tmp |= src << 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 1U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 1U; - src = *(in + 16 * 34 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 30U; - src = *(in + 16 * 35 + i); - tmp |= src << 3U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 5U; - src = *(in + 16 * 38 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 26U; - src = *(in + 16 * 39 + i); - tmp |= src << 7U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 9U; - src = *(in + 16 * 42 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 22U; - src = *(in + 16 * 43 + i); - tmp |= src << 11U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 13U; - src = *(in + 16 * 46 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 18U; - src = *(in + 16 * 47 + i); - tmp |= src << 15U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 17U; - src = *(in + 16 * 50 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 14U; - src = *(in + 16 * 51 + i); - tmp |= src << 19U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 21U; - src = *(in + 16 * 54 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 10U; - src = *(in + 16 * 55 + i); - tmp |= src << 23U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 25U; - src = *(in + 16 * 58 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 6U; - src = *(in + 16 * 59 + i); - tmp |= src << 27U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 29U; - src = *(in + 16 * 62 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 2U; - src = *(in + 16 * 63 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out -= 512; - } -} -void pack_34bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 30U; - src = *(in + 16 * 2 + i); - tmp |= src << 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 26U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 22U; - src = *(in + 16 * 6 + i); - tmp |= src << 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 18U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 14U; - src = *(in + 16 * 10 + i); - tmp |= src << 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 10U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 6U; - src = *(in + 16 * 14 + i); - tmp |= src << 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 2U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 2U; - src = *(in + 16 * 18 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 28U; - src = *(in + 16 * 19 + i); - tmp |= src << 6U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 10U; - src = *(in + 16 * 22 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 20U; - src = *(in + 16 * 23 + i); - tmp |= src << 14U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 18U; - src = *(in + 16 * 26 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 12U; - src = *(in + 16 * 27 + i); - tmp |= src << 22U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 26U; - src = *(in + 16 * 30 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 4U; - src = *(in + 16 * 31 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 30U; - src = *(in + 16 * 34 + i); - tmp |= src << 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 26U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 22U; - src = *(in + 16 * 38 + i); - tmp |= src << 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 18U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 14U; - src = *(in + 16 * 42 + i); - tmp |= src << 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 10U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 6U; - src = *(in + 16 * 46 + i); - tmp |= src << 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 2U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 2U; - src = *(in + 16 * 50 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 28U; - src = *(in + 16 * 51 + i); - tmp |= src << 6U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 10U; - src = *(in + 16 * 54 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 20U; - src = *(in + 16 * 55 + i); - tmp |= src << 14U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 18U; - src = *(in + 16 * 58 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 12U; - src = *(in + 16 * 59 + i); - tmp |= src << 22U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 26U; - src = *(in + 16 * 62 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 4U; - src = *(in + 16 * 63 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out -= 528; - } -} -void pack_35bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 29U; - src = *(in + 16 * 2 + i); - tmp |= src << 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 23U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 17U; - src = *(in + 16 * 6 + i); - tmp |= src << 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 11U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 5U; - src = *(in + 16 * 10 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 34U; - src = *(in + 16 * 11 + i); - tmp |= src << 1U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 7U; - src = *(in + 16 * 14 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 22U; - src = *(in + 16 * 15 + i); - tmp |= src << 13U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 19U; - src = *(in + 16 * 18 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 10U; - src = *(in + 16 * 19 + i); - tmp |= src << 25U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 33U; - src = *(in + 16 * 22 + i); - tmp |= src << 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 27U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 21U; - src = *(in + 16 * 26 + i); - tmp |= src << 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 15U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 9U; - src = *(in + 16 * 30 + i); - tmp |= src << 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 3U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 3U; - src = *(in + 16 * 34 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 26U; - src = *(in + 16 * 35 + i); - tmp |= src << 9U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 15U; - src = *(in + 16 * 38 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 14U; - src = *(in + 16 * 39 + i); - tmp |= src << 21U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 27U; - src = *(in + 16 * 42 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 31U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 25U; - src = *(in + 16 * 46 + i); - tmp |= src << 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 19U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 13U; - src = *(in + 16 * 50 + i); - tmp |= src << 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 7U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 1U; - src = *(in + 16 * 54 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 30U; - src = *(in + 16 * 55 + i); - tmp |= src << 5U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 11U; - src = *(in + 16 * 58 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 18U; - src = *(in + 16 * 59 + i); - tmp |= src << 17U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 23U; - src = *(in + 16 * 62 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 6U; - src = *(in + 16 * 63 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out -= 544; - } -} -void pack_36bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 28U; - src = *(in + 16 * 2 + i); - tmp |= src << 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 20U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 12U; - src = *(in + 16 * 6 + i); - tmp |= src << 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 4U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 4U; - src = *(in + 16 * 10 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 24U; - src = *(in + 16 * 11 + i); - tmp |= src << 12U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 20U; - src = *(in + 16 * 14 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 8U; - src = *(in + 16 * 15 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 28U; - src = *(in + 16 * 18 + i); - tmp |= src << 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 20U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 12U; - src = *(in + 16 * 22 + i); - tmp |= src << 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 4U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 4U; - src = *(in + 16 * 26 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 24U; - src = *(in + 16 * 27 + i); - tmp |= src << 12U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 20U; - src = *(in + 16 * 30 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 8U; - src = *(in + 16 * 31 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 28U; - src = *(in + 16 * 34 + i); - tmp |= src << 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 20U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 12U; - src = *(in + 16 * 38 + i); - tmp |= src << 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 4U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 4U; - src = *(in + 16 * 42 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 24U; - src = *(in + 16 * 43 + i); - tmp |= src << 12U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 20U; - src = *(in + 16 * 46 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 8U; - src = *(in + 16 * 47 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 28U; - src = *(in + 16 * 50 + i); - tmp |= src << 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 20U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 12U; - src = *(in + 16 * 54 + i); - tmp |= src << 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 4U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 4U; - src = *(in + 16 * 58 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 24U; - src = *(in + 16 * 59 + i); - tmp |= src << 12U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 20U; - src = *(in + 16 * 62 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 8U; - src = *(in + 16 * 63 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out -= 560; - } -} -void pack_37bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 27U; - src = *(in + 16 * 2 + i); - tmp |= src << 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 17U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 7U; - src = *(in + 16 * 6 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 34U; - src = *(in + 16 * 7 + i); - tmp |= src << 3U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 13U; - src = *(in + 16 * 10 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 14U; - src = *(in + 16 * 11 + i); - tmp |= src << 23U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 31U; - src = *(in + 16 * 14 + i); - tmp |= src << 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 21U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 11U; - src = *(in + 16 * 18 + i); - tmp |= src << 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 1U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 9U; - src = *(in + 16 * 22 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 18U; - src = *(in + 16 * 23 + i); - tmp |= src << 19U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 35U; - src = *(in + 16 * 26 + i); - tmp |= src << 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 25U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 15U; - src = *(in + 16 * 30 + i); - tmp |= src << 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 5U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 5U; - src = *(in + 16 * 34 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 22U; - src = *(in + 16 * 35 + i); - tmp |= src << 15U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 25U; - src = *(in + 16 * 38 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 29U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 19U; - src = *(in + 16 * 42 + i); - tmp |= src << 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 9U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 1U; - src = *(in + 16 * 46 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 26U; - src = *(in + 16 * 47 + i); - tmp |= src << 11U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 21U; - src = *(in + 16 * 50 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 6U; - src = *(in + 16 * 51 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 33U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 23U; - src = *(in + 16 * 54 + i); - tmp |= src << 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 13U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 3U; - src = *(in + 16 * 58 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 30U; - src = *(in + 16 * 59 + i); - tmp |= src << 7U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 17U; - src = *(in + 16 * 62 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 10U; - src = *(in + 16 * 63 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out -= 576; - } -} -void pack_38bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 26U; - src = *(in + 16 * 2 + i); - tmp |= src << 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 14U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 2U; - src = *(in + 16 * 6 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 28U; - src = *(in + 16 * 7 + i); - tmp |= src << 10U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 22U; - src = *(in + 16 * 10 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 30U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 18U; - src = *(in + 16 * 14 + i); - tmp |= src << 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 6U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 6U; - src = *(in + 16 * 18 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 20U; - src = *(in + 16 * 19 + i); - tmp |= src << 18U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 34U; - src = *(in + 16 * 22 + i); - tmp |= src << 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 22U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 10U; - src = *(in + 16 * 26 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 36U; - src = *(in + 16 * 27 + i); - tmp |= src << 2U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 14U; - src = *(in + 16 * 30 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 12U; - src = *(in + 16 * 31 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 26U; - src = *(in + 16 * 34 + i); - tmp |= src << 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 14U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 2U; - src = *(in + 16 * 38 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 28U; - src = *(in + 16 * 39 + i); - tmp |= src << 10U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 22U; - src = *(in + 16 * 42 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 30U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 18U; - src = *(in + 16 * 46 + i); - tmp |= src << 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 6U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 6U; - src = *(in + 16 * 50 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 20U; - src = *(in + 16 * 51 + i); - tmp |= src << 18U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 34U; - src = *(in + 16 * 54 + i); - tmp |= src << 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 22U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 10U; - src = *(in + 16 * 58 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 36U; - src = *(in + 16 * 59 + i); - tmp |= src << 2U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 14U; - src = *(in + 16 * 62 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 12U; - src = *(in + 16 * 63 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out -= 592; - } -} -void pack_39bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 25U; - src = *(in + 16 * 2 + i); - tmp |= src << 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 11U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 3U; - src = *(in + 16 * 6 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 22U; - src = *(in + 16 * 7 + i); - tmp |= src << 17U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 33U; - src = *(in + 16 * 10 + i); - tmp |= src << 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 19U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 5U; - src = *(in + 16 * 14 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 30U; - src = *(in + 16 * 15 + i); - tmp |= src << 9U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 23U; - src = *(in + 16 * 18 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 27U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 13U; - src = *(in + 16 * 22 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 38U; - src = *(in + 16 * 23 + i); - tmp |= src << 1U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 15U; - src = *(in + 16 * 26 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 10U; - src = *(in + 16 * 27 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 35U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 21U; - src = *(in + 16 * 30 + i); - tmp |= src << 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 7U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 7U; - src = *(in + 16 * 34 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 18U; - src = *(in + 16 * 35 + i); - tmp |= src << 21U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 29U; - src = *(in + 16 * 38 + i); - tmp |= src << 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 15U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 1U; - src = *(in + 16 * 42 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 26U; - src = *(in + 16 * 43 + i); - tmp |= src << 13U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 37U; - src = *(in + 16 * 46 + i); - tmp |= src << 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 23U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 9U; - src = *(in + 16 * 50 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 34U; - src = *(in + 16 * 51 + i); - tmp |= src << 5U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 19U; - src = *(in + 16 * 54 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 31U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 17U; - src = *(in + 16 * 58 + i); - tmp |= src << 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 3U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 11U; - src = *(in + 16 * 62 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 14U; - src = *(in + 16 * 63 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out -= 608; - } -} -void pack_40bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 24U; - src = *(in + 16 * 2 + i); - tmp |= src << 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 8U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 8U; - src = *(in + 16 * 6 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 16U; - src = *(in + 16 * 7 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 24U; - src = *(in + 16 * 10 + i); - tmp |= src << 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 8U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 8U; - src = *(in + 16 * 14 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 16U; - src = *(in + 16 * 15 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 24U; - src = *(in + 16 * 18 + i); - tmp |= src << 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 8U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 8U; - src = *(in + 16 * 22 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 16U; - src = *(in + 16 * 23 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 24U; - src = *(in + 16 * 26 + i); - tmp |= src << 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 8U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 8U; - src = *(in + 16 * 30 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 16U; - src = *(in + 16 * 31 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 24U; - src = *(in + 16 * 34 + i); - tmp |= src << 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 8U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 8U; - src = *(in + 16 * 38 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 16U; - src = *(in + 16 * 39 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 24U; - src = *(in + 16 * 42 + i); - tmp |= src << 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 8U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 8U; - src = *(in + 16 * 46 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 16U; - src = *(in + 16 * 47 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 24U; - src = *(in + 16 * 50 + i); - tmp |= src << 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 8U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 8U; - src = *(in + 16 * 54 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 16U; - src = *(in + 16 * 55 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 24U; - src = *(in + 16 * 58 + i); - tmp |= src << 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 8U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 8U; - src = *(in + 16 * 62 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 16U; - src = *(in + 16 * 63 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out -= 624; - } -} -void pack_41bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 23U; - src = *(in + 16 * 2 + i); - tmp |= src << 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 5U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 13U; - src = *(in + 16 * 6 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 10U; - src = *(in + 16 * 7 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 33U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 15U; - src = *(in + 16 * 10 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 38U; - src = *(in + 16 * 11 + i); - tmp |= src << 3U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 21U; - src = *(in + 16 * 14 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 25U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 7U; - src = *(in + 16 * 18 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 30U; - src = *(in + 16 * 19 + i); - tmp |= src << 11U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 35U; - src = *(in + 16 * 22 + i); - tmp |= src << 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 17U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 1U; - src = *(in + 16 * 26 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 22U; - src = *(in + 16 * 27 + i); - tmp |= src << 19U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 27U; - src = *(in + 16 * 30 + i); - tmp |= src << 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 9U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 9U; - src = *(in + 16 * 34 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 14U; - src = *(in + 16 * 35 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 37U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 19U; - src = *(in + 16 * 38 + i); - tmp |= src << 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 1U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 17U; - src = *(in + 16 * 42 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 29U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 11U; - src = *(in + 16 * 46 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 34U; - src = *(in + 16 * 47 + i); - tmp |= src << 7U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 39U; - src = *(in + 16 * 50 + i); - tmp |= src << 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 21U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 3U; - src = *(in + 16 * 54 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 26U; - src = *(in + 16 * 55 + i); - tmp |= src << 15U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 31U; - src = *(in + 16 * 58 + i); - tmp |= src << 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 13U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 5U; - src = *(in + 16 * 62 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 18U; - src = *(in + 16 * 63 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out -= 640; - } -} -void pack_42bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 22U; - src = *(in + 16 * 2 + i); - tmp |= src << 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 2U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 18U; - src = *(in + 16 * 6 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 26U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 6U; - src = *(in + 16 * 10 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 28U; - src = *(in + 16 * 11 + i); - tmp |= src << 14U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 30U; - src = *(in + 16 * 14 + i); - tmp |= src << 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 10U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 10U; - src = *(in + 16 * 18 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 12U; - src = *(in + 16 * 19 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 34U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 14U; - src = *(in + 16 * 22 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 36U; - src = *(in + 16 * 23 + i); - tmp |= src << 6U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 38U; - src = *(in + 16 * 26 + i); - tmp |= src << 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 18U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 2U; - src = *(in + 16 * 30 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 20U; - src = *(in + 16 * 31 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 22U; - src = *(in + 16 * 34 + i); - tmp |= src << 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 2U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 18U; - src = *(in + 16 * 38 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 26U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 6U; - src = *(in + 16 * 42 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 28U; - src = *(in + 16 * 43 + i); - tmp |= src << 14U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 30U; - src = *(in + 16 * 46 + i); - tmp |= src << 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 10U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 10U; - src = *(in + 16 * 50 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 12U; - src = *(in + 16 * 51 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 34U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 14U; - src = *(in + 16 * 54 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 36U; - src = *(in + 16 * 55 + i); - tmp |= src << 6U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 38U; - src = *(in + 16 * 58 + i); - tmp |= src << 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 18U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 2U; - src = *(in + 16 * 62 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 20U; - src = *(in + 16 * 63 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out -= 656; - } -} -void pack_43bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 21U; - src = *(in + 16 * 2 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 42U; - src = *(in + 16 * 3 + i); - tmp |= src << 1U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 41U; - src = *(in + 16 * 6 + i); - tmp |= src << 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 19U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 3U; - src = *(in + 16 * 10 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 18U; - src = *(in + 16 * 11 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 39U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 17U; - src = *(in + 16 * 14 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 38U; - src = *(in + 16 * 15 + i); - tmp |= src << 5U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 37U; - src = *(in + 16 * 18 + i); - tmp |= src << 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 15U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 7U; - src = *(in + 16 * 22 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 14U; - src = *(in + 16 * 23 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 35U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 13U; - src = *(in + 16 * 26 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 34U; - src = *(in + 16 * 27 + i); - tmp |= src << 9U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 33U; - src = *(in + 16 * 30 + i); - tmp |= src << 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 11U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 11U; - src = *(in + 16 * 34 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 31U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 9U; - src = *(in + 16 * 38 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 30U; - src = *(in + 16 * 39 + i); - tmp |= src << 13U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 29U; - src = *(in + 16 * 42 + i); - tmp |= src << 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 7U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 15U; - src = *(in + 16 * 46 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 27U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 5U; - src = *(in + 16 * 50 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 26U; - src = *(in + 16 * 51 + i); - tmp |= src << 17U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 25U; - src = *(in + 16 * 54 + i); - tmp |= src << 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 3U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 19U; - src = *(in + 16 * 58 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 23U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 1U; - src = *(in + 16 * 62 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 22U; - src = *(in + 16 * 63 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out -= 672; - } -} -void pack_44bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 20U; - src = *(in + 16 * 2 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 40U; - src = *(in + 16 * 3 + i); - tmp |= src << 4U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 36U; - src = *(in + 16 * 6 + i); - tmp |= src << 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 12U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 12U; - src = *(in + 16 * 10 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 28U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 4U; - src = *(in + 16 * 14 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 24U; - src = *(in + 16 * 15 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 20U; - src = *(in + 16 * 18 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 40U; - src = *(in + 16 * 19 + i); - tmp |= src << 4U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 36U; - src = *(in + 16 * 22 + i); - tmp |= src << 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 12U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 12U; - src = *(in + 16 * 26 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 28U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 4U; - src = *(in + 16 * 30 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 24U; - src = *(in + 16 * 31 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 20U; - src = *(in + 16 * 34 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 40U; - src = *(in + 16 * 35 + i); - tmp |= src << 4U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 36U; - src = *(in + 16 * 38 + i); - tmp |= src << 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 12U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 12U; - src = *(in + 16 * 42 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 28U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 4U; - src = *(in + 16 * 46 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 24U; - src = *(in + 16 * 47 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 20U; - src = *(in + 16 * 50 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 40U; - src = *(in + 16 * 51 + i); - tmp |= src << 4U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 36U; - src = *(in + 16 * 54 + i); - tmp |= src << 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 12U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 12U; - src = *(in + 16 * 58 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 28U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 4U; - src = *(in + 16 * 62 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 24U; - src = *(in + 16 * 63 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out -= 688; - } -} -void pack_45bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 19U; - src = *(in + 16 * 2 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 38U; - src = *(in + 16 * 3 + i); - tmp |= src << 7U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 31U; - src = *(in + 16 * 6 + i); - tmp |= src << 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 5U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 43U; - src = *(in + 16 * 10 + i); - tmp |= src << 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 17U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 9U; - src = *(in + 16 * 14 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 29U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 3U; - src = *(in + 16 * 18 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 22U; - src = *(in + 16 * 19 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 41U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 15U; - src = *(in + 16 * 22 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 34U; - src = *(in + 16 * 23 + i); - tmp |= src << 11U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 27U; - src = *(in + 16 * 26 + i); - tmp |= src << 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 1U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 39U; - src = *(in + 16 * 30 + i); - tmp |= src << 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 13U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 13U; - src = *(in + 16 * 34 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 25U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 1U; - src = *(in + 16 * 38 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 18U; - src = *(in + 16 * 39 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 37U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 11U; - src = *(in + 16 * 42 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 30U; - src = *(in + 16 * 43 + i); - tmp |= src << 15U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 23U; - src = *(in + 16 * 46 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 42U; - src = *(in + 16 * 47 + i); - tmp |= src << 3U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 35U; - src = *(in + 16 * 50 + i); - tmp |= src << 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 9U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 17U; - src = *(in + 16 * 54 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 21U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 5U; - src = *(in + 16 * 58 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 14U; - src = *(in + 16 * 59 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 33U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 7U; - src = *(in + 16 * 62 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 26U; - src = *(in + 16 * 63 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out -= 704; - } -} -void pack_46bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 18U; - src = *(in + 16 * 2 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 36U; - src = *(in + 16 * 3 + i); - tmp |= src << 10U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 26U; - src = *(in + 16 * 6 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 44U; - src = *(in + 16 * 7 + i); - tmp |= src << 2U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 34U; - src = *(in + 16 * 10 + i); - tmp |= src << 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 6U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 42U; - src = *(in + 16 * 14 + i); - tmp |= src << 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 14U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 14U; - src = *(in + 16 * 18 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 22U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 6U; - src = *(in + 16 * 22 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 30U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 2U; - src = *(in + 16 * 26 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 20U; - src = *(in + 16 * 27 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 38U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 10U; - src = *(in + 16 * 30 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 28U; - src = *(in + 16 * 31 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 18U; - src = *(in + 16 * 34 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 36U; - src = *(in + 16 * 35 + i); - tmp |= src << 10U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 26U; - src = *(in + 16 * 38 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 44U; - src = *(in + 16 * 39 + i); - tmp |= src << 2U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 34U; - src = *(in + 16 * 42 + i); - tmp |= src << 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 6U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 42U; - src = *(in + 16 * 46 + i); - tmp |= src << 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 14U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 14U; - src = *(in + 16 * 50 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 22U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 6U; - src = *(in + 16 * 54 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 30U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 2U; - src = *(in + 16 * 58 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 20U; - src = *(in + 16 * 59 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 38U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 10U; - src = *(in + 16 * 62 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 28U; - src = *(in + 16 * 63 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out -= 720; - } -} -void pack_47bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 17U; - src = *(in + 16 * 2 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 34U; - src = *(in + 16 * 3 + i); - tmp |= src << 13U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 21U; - src = *(in + 16 * 6 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 38U; - src = *(in + 16 * 7 + i); - tmp |= src << 9U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 25U; - src = *(in + 16 * 10 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 42U; - src = *(in + 16 * 11 + i); - tmp |= src << 5U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 29U; - src = *(in + 16 * 14 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 46U; - src = *(in + 16 * 15 + i); - tmp |= src << 1U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 33U; - src = *(in + 16 * 18 + i); - tmp |= src << 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 3U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 37U; - src = *(in + 16 * 22 + i); - tmp |= src << 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 7U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 41U; - src = *(in + 16 * 26 + i); - tmp |= src << 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 11U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 45U; - src = *(in + 16 * 30 + i); - tmp |= src << 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 15U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 15U; - src = *(in + 16 * 34 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 19U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 11U; - src = *(in + 16 * 38 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 23U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 7U; - src = *(in + 16 * 42 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 27U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 3U; - src = *(in + 16 * 46 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 31U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 1U; - src = *(in + 16 * 50 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 18U; - src = *(in + 16 * 51 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 35U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 5U; - src = *(in + 16 * 54 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 22U; - src = *(in + 16 * 55 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 39U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 9U; - src = *(in + 16 * 58 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 26U; - src = *(in + 16 * 59 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 43U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 13U; - src = *(in + 16 * 62 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 30U; - src = *(in + 16 * 63 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out -= 736; - } -} -void pack_48bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 16U; - src = *(in + 16 * 2 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 32U; - src = *(in + 16 * 3 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - src = *(in + 16 * 5 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 16U; - src = *(in + 16 * 6 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 32U; - src = *(in + 16 * 7 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 16U; - src = *(in + 16 * 10 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 32U; - src = *(in + 16 * 11 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - src = *(in + 16 * 13 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 16U; - src = *(in + 16 * 14 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 32U; - src = *(in + 16 * 15 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 16U; - src = *(in + 16 * 18 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 32U; - src = *(in + 16 * 19 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - src = *(in + 16 * 21 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 16U; - src = *(in + 16 * 22 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 32U; - src = *(in + 16 * 23 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 16U; - src = *(in + 16 * 26 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 32U; - src = *(in + 16 * 27 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - src = *(in + 16 * 29 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 16U; - src = *(in + 16 * 30 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 32U; - src = *(in + 16 * 31 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 16U; - src = *(in + 16 * 34 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 32U; - src = *(in + 16 * 35 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - src = *(in + 16 * 37 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 16U; - src = *(in + 16 * 38 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 32U; - src = *(in + 16 * 39 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 16U; - src = *(in + 16 * 42 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 32U; - src = *(in + 16 * 43 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - src = *(in + 16 * 45 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 16U; - src = *(in + 16 * 46 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 32U; - src = *(in + 16 * 47 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 16U; - src = *(in + 16 * 50 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 32U; - src = *(in + 16 * 51 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - src = *(in + 16 * 53 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 16U; - src = *(in + 16 * 54 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 32U; - src = *(in + 16 * 55 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 16U; - src = *(in + 16 * 58 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 32U; - src = *(in + 16 * 59 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - src = *(in + 16 * 61 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 16U; - src = *(in + 16 * 62 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 32U; - src = *(in + 16 * 63 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out -= 752; - } -} -void pack_49bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 15U; - src = *(in + 16 * 2 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 30U; - src = *(in + 16 * 3 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 45U; - src = *(in + 16 * 4 + i); - tmp |= src << 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 11U; - src = *(in + 16 * 6 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 26U; - src = *(in + 16 * 7 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 41U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 7U; - src = *(in + 16 * 10 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 22U; - src = *(in + 16 * 11 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 37U; - src = *(in + 16 * 12 + i); - tmp |= src << 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 3U; - src = *(in + 16 * 14 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 18U; - src = *(in + 16 * 15 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 33U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 1U; - src = *(in + 16 * 18 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 14U; - src = *(in + 16 * 19 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 29U; - src = *(in + 16 * 20 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 44U; - src = *(in + 16 * 21 + i); - tmp |= src << 5U; - src = *(in + 16 * 22 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 10U; - src = *(in + 16 * 23 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 25U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 9U; - src = *(in + 16 * 26 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 6U; - src = *(in + 16 * 27 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 21U; - src = *(in + 16 * 28 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 36U; - src = *(in + 16 * 29 + i); - tmp |= src << 13U; - src = *(in + 16 * 30 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 2U; - src = *(in + 16 * 31 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 17U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 47U; - src = *(in + 16 * 34 + i); - tmp |= src << 2U; - src = *(in + 16 * 35 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 13U; - src = *(in + 16 * 36 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 28U; - src = *(in + 16 * 37 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 43U; - src = *(in + 16 * 38 + i); - tmp |= src << 6U; - src = *(in + 16 * 39 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 9U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 39U; - src = *(in + 16 * 42 + i); - tmp |= src << 10U; - src = *(in + 16 * 43 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 5U; - src = *(in + 16 * 44 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 20U; - src = *(in + 16 * 45 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 35U; - src = *(in + 16 * 46 + i); - tmp |= src << 14U; - src = *(in + 16 * 47 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 1U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 31U; - src = *(in + 16 * 50 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 46U; - src = *(in + 16 * 51 + i); - tmp |= src << 3U; - src = *(in + 16 * 52 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 12U; - src = *(in + 16 * 53 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 27U; - src = *(in + 16 * 54 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 42U; - src = *(in + 16 * 55 + i); - tmp |= src << 7U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 23U; - src = *(in + 16 * 58 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 38U; - src = *(in + 16 * 59 + i); - tmp |= src << 11U; - src = *(in + 16 * 60 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 4U; - src = *(in + 16 * 61 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 19U; - src = *(in + 16 * 62 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 34U; - src = *(in + 16 * 63 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out -= 768; - } -} -void pack_50bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 14U; - src = *(in + 16 * 2 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 28U; - src = *(in + 16 * 3 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 42U; - src = *(in + 16 * 4 + i); - tmp |= src << 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 6U; - src = *(in + 16 * 6 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 20U; - src = *(in + 16 * 7 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 34U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 2U; - src = *(in + 16 * 10 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 12U; - src = *(in + 16 * 11 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 26U; - src = *(in + 16 * 12 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 40U; - src = *(in + 16 * 13 + i); - tmp |= src << 10U; - src = *(in + 16 * 14 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 4U; - src = *(in + 16 * 15 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 18U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 46U; - src = *(in + 16 * 18 + i); - tmp |= src << 4U; - src = *(in + 16 * 19 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 10U; - src = *(in + 16 * 20 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 24U; - src = *(in + 16 * 21 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 38U; - src = *(in + 16 * 22 + i); - tmp |= src << 12U; - src = *(in + 16 * 23 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 2U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 30U; - src = *(in + 16 * 26 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 44U; - src = *(in + 16 * 27 + i); - tmp |= src << 6U; - src = *(in + 16 * 28 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 8U; - src = *(in + 16 * 29 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 22U; - src = *(in + 16 * 30 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 36U; - src = *(in + 16 * 31 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 14U; - src = *(in + 16 * 34 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 28U; - src = *(in + 16 * 35 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 42U; - src = *(in + 16 * 36 + i); - tmp |= src << 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 6U; - src = *(in + 16 * 38 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 20U; - src = *(in + 16 * 39 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 34U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 2U; - src = *(in + 16 * 42 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 12U; - src = *(in + 16 * 43 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 26U; - src = *(in + 16 * 44 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 40U; - src = *(in + 16 * 45 + i); - tmp |= src << 10U; - src = *(in + 16 * 46 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 4U; - src = *(in + 16 * 47 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 18U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 46U; - src = *(in + 16 * 50 + i); - tmp |= src << 4U; - src = *(in + 16 * 51 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 10U; - src = *(in + 16 * 52 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 24U; - src = *(in + 16 * 53 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 38U; - src = *(in + 16 * 54 + i); - tmp |= src << 12U; - src = *(in + 16 * 55 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 2U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 30U; - src = *(in + 16 * 58 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 44U; - src = *(in + 16 * 59 + i); - tmp |= src << 6U; - src = *(in + 16 * 60 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 8U; - src = *(in + 16 * 61 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 22U; - src = *(in + 16 * 62 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 36U; - src = *(in + 16 * 63 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out -= 784; - } -} -void pack_51bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 13U; - src = *(in + 16 * 2 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 26U; - src = *(in + 16 * 3 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 39U; - src = *(in + 16 * 4 + i); - tmp |= src << 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 1U; - src = *(in + 16 * 6 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 14U; - src = *(in + 16 * 7 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 27U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 11U; - src = *(in + 16 * 10 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 2U; - src = *(in + 16 * 11 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 15U; - src = *(in + 16 * 12 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 28U; - src = *(in + 16 * 13 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 41U; - src = *(in + 16 * 14 + i); - tmp |= src << 10U; - src = *(in + 16 * 15 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 3U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 29U; - src = *(in + 16 * 18 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 42U; - src = *(in + 16 * 19 + i); - tmp |= src << 9U; - src = *(in + 16 * 20 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 4U; - src = *(in + 16 * 21 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 17U; - src = *(in + 16 * 22 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 30U; - src = *(in + 16 * 23 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 43U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 5U; - src = *(in + 16 * 26 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 18U; - src = *(in + 16 * 27 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 31U; - src = *(in + 16 * 28 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 44U; - src = *(in + 16 * 29 + i); - tmp |= src << 7U; - src = *(in + 16 * 30 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 6U; - src = *(in + 16 * 31 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 19U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 45U; - src = *(in + 16 * 34 + i); - tmp |= src << 6U; - src = *(in + 16 * 35 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 7U; - src = *(in + 16 * 36 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 20U; - src = *(in + 16 * 37 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 33U; - src = *(in + 16 * 38 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 46U; - src = *(in + 16 * 39 + i); - tmp |= src << 5U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 21U; - src = *(in + 16 * 42 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 34U; - src = *(in + 16 * 43 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 47U; - src = *(in + 16 * 44 + i); - tmp |= src << 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 9U; - src = *(in + 16 * 46 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 22U; - src = *(in + 16 * 47 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 35U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 3U; - src = *(in + 16 * 50 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 10U; - src = *(in + 16 * 51 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 23U; - src = *(in + 16 * 52 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 36U; - src = *(in + 16 * 53 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 49U; - src = *(in + 16 * 54 + i); - tmp |= src << 2U; - src = *(in + 16 * 55 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 11U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 37U; - src = *(in + 16 * 58 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 50U; - src = *(in + 16 * 59 + i); - tmp |= src << 1U; - src = *(in + 16 * 60 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 12U; - src = *(in + 16 * 61 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 25U; - src = *(in + 16 * 62 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 38U; - src = *(in + 16 * 63 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out -= 800; - } -} -void pack_52bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 12U; - src = *(in + 16 * 2 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 24U; - src = *(in + 16 * 3 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 36U; - src = *(in + 16 * 4 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 48U; - src = *(in + 16 * 5 + i); - tmp |= src << 4U; - src = *(in + 16 * 6 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 8U; - src = *(in + 16 * 7 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 20U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 44U; - src = *(in + 16 * 10 + i); - tmp |= src << 8U; - src = *(in + 16 * 11 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 4U; - src = *(in + 16 * 12 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 16U; - src = *(in + 16 * 13 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 28U; - src = *(in + 16 * 14 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 40U; - src = *(in + 16 * 15 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 12U; - src = *(in + 16 * 18 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 24U; - src = *(in + 16 * 19 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 36U; - src = *(in + 16 * 20 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 48U; - src = *(in + 16 * 21 + i); - tmp |= src << 4U; - src = *(in + 16 * 22 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 8U; - src = *(in + 16 * 23 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 20U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 44U; - src = *(in + 16 * 26 + i); - tmp |= src << 8U; - src = *(in + 16 * 27 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 4U; - src = *(in + 16 * 28 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 16U; - src = *(in + 16 * 29 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 28U; - src = *(in + 16 * 30 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 40U; - src = *(in + 16 * 31 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 12U; - src = *(in + 16 * 34 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 24U; - src = *(in + 16 * 35 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 36U; - src = *(in + 16 * 36 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 48U; - src = *(in + 16 * 37 + i); - tmp |= src << 4U; - src = *(in + 16 * 38 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 8U; - src = *(in + 16 * 39 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 20U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 44U; - src = *(in + 16 * 42 + i); - tmp |= src << 8U; - src = *(in + 16 * 43 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 4U; - src = *(in + 16 * 44 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 16U; - src = *(in + 16 * 45 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 28U; - src = *(in + 16 * 46 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 40U; - src = *(in + 16 * 47 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 12U; - src = *(in + 16 * 50 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 24U; - src = *(in + 16 * 51 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 36U; - src = *(in + 16 * 52 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 48U; - src = *(in + 16 * 53 + i); - tmp |= src << 4U; - src = *(in + 16 * 54 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 8U; - src = *(in + 16 * 55 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 20U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 44U; - src = *(in + 16 * 58 + i); - tmp |= src << 8U; - src = *(in + 16 * 59 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 4U; - src = *(in + 16 * 60 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 16U; - src = *(in + 16 * 61 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 28U; - src = *(in + 16 * 62 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 40U; - src = *(in + 16 * 63 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out -= 816; - } -} -void pack_53bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 11U; - src = *(in + 16 * 2 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 22U; - src = *(in + 16 * 3 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 33U; - src = *(in + 16 * 4 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 44U; - src = *(in + 16 * 5 + i); - tmp |= src << 9U; - src = *(in + 16 * 6 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 2U; - src = *(in + 16 * 7 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 13U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 35U; - src = *(in + 16 * 10 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 46U; - src = *(in + 16 * 11 + i); - tmp |= src << 7U; - src = *(in + 16 * 12 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 15U; - src = *(in + 16 * 14 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 26U; - src = *(in + 16 * 15 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 37U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 5U; - src = *(in + 16 * 18 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 6U; - src = *(in + 16 * 19 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 17U; - src = *(in + 16 * 20 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 28U; - src = *(in + 16 * 21 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 39U; - src = *(in + 16 * 22 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 50U; - src = *(in + 16 * 23 + i); - tmp |= src << 3U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 19U; - src = *(in + 16 * 26 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 30U; - src = *(in + 16 * 27 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 41U; - src = *(in + 16 * 28 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 52U; - src = *(in + 16 * 29 + i); - tmp |= src << 1U; - src = *(in + 16 * 30 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 10U; - src = *(in + 16 * 31 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 21U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 43U; - src = *(in + 16 * 34 + i); - tmp |= src << 10U; - src = *(in + 16 * 35 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 1U; - src = *(in + 16 * 36 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 12U; - src = *(in + 16 * 37 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 23U; - src = *(in + 16 * 38 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 34U; - src = *(in + 16 * 39 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 45U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 3U; - src = *(in + 16 * 42 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 14U; - src = *(in + 16 * 43 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 25U; - src = *(in + 16 * 44 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 36U; - src = *(in + 16 * 45 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 47U; - src = *(in + 16 * 46 + i); - tmp |= src << 6U; - src = *(in + 16 * 47 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 5U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 27U; - src = *(in + 16 * 50 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 38U; - src = *(in + 16 * 51 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 49U; - src = *(in + 16 * 52 + i); - tmp |= src << 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 7U; - src = *(in + 16 * 54 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 18U; - src = *(in + 16 * 55 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 29U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 51U; - src = *(in + 16 * 58 + i); - tmp |= src << 2U; - src = *(in + 16 * 59 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 9U; - src = *(in + 16 * 60 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 20U; - src = *(in + 16 * 61 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 31U; - src = *(in + 16 * 62 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 42U; - src = *(in + 16 * 63 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out -= 832; - } -} -void pack_54bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 10U; - src = *(in + 16 * 2 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 20U; - src = *(in + 16 * 3 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 30U; - src = *(in + 16 * 4 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 40U; - src = *(in + 16 * 5 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 50U; - src = *(in + 16 * 6 + i); - tmp |= src << 4U; - src = *(in + 16 * 7 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 6U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 26U; - src = *(in + 16 * 10 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 36U; - src = *(in + 16 * 11 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 46U; - src = *(in + 16 * 12 + i); - tmp |= src << 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 2U; - src = *(in + 16 * 14 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 12U; - src = *(in + 16 * 15 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 22U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 42U; - src = *(in + 16 * 18 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 52U; - src = *(in + 16 * 19 + i); - tmp |= src << 2U; - src = *(in + 16 * 20 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 8U; - src = *(in + 16 * 21 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 18U; - src = *(in + 16 * 22 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 28U; - src = *(in + 16 * 23 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 38U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 6U; - src = *(in + 16 * 26 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 4U; - src = *(in + 16 * 27 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 14U; - src = *(in + 16 * 28 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 24U; - src = *(in + 16 * 29 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 34U; - src = *(in + 16 * 30 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 44U; - src = *(in + 16 * 31 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 10U; - src = *(in + 16 * 34 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 20U; - src = *(in + 16 * 35 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 30U; - src = *(in + 16 * 36 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 40U; - src = *(in + 16 * 37 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 50U; - src = *(in + 16 * 38 + i); - tmp |= src << 4U; - src = *(in + 16 * 39 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 6U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 26U; - src = *(in + 16 * 42 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 36U; - src = *(in + 16 * 43 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 46U; - src = *(in + 16 * 44 + i); - tmp |= src << 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 2U; - src = *(in + 16 * 46 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 12U; - src = *(in + 16 * 47 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 22U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 42U; - src = *(in + 16 * 50 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 52U; - src = *(in + 16 * 51 + i); - tmp |= src << 2U; - src = *(in + 16 * 52 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 8U; - src = *(in + 16 * 53 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 18U; - src = *(in + 16 * 54 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 28U; - src = *(in + 16 * 55 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 38U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 6U; - src = *(in + 16 * 58 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 4U; - src = *(in + 16 * 59 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 14U; - src = *(in + 16 * 60 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 24U; - src = *(in + 16 * 61 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 34U; - src = *(in + 16 * 62 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 44U; - src = *(in + 16 * 63 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out -= 848; - } -} -void pack_55bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 9U; - src = *(in + 16 * 2 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 18U; - src = *(in + 16 * 3 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 27U; - src = *(in + 16 * 4 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 36U; - src = *(in + 16 * 5 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 45U; - src = *(in + 16 * 6 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 54U; - src = *(in + 16 * 7 + i); - tmp |= src << 1U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 17U; - src = *(in + 16 * 10 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 26U; - src = *(in + 16 * 11 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 35U; - src = *(in + 16 * 12 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 44U; - src = *(in + 16 * 13 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 53U; - src = *(in + 16 * 14 + i); - tmp |= src << 2U; - src = *(in + 16 * 15 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 7U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 25U; - src = *(in + 16 * 18 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 34U; - src = *(in + 16 * 19 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 43U; - src = *(in + 16 * 20 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 52U; - src = *(in + 16 * 21 + i); - tmp |= src << 3U; - src = *(in + 16 * 22 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 6U; - src = *(in + 16 * 23 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 15U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 33U; - src = *(in + 16 * 26 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 42U; - src = *(in + 16 * 27 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 51U; - src = *(in + 16 * 28 + i); - tmp |= src << 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 5U; - src = *(in + 16 * 30 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 14U; - src = *(in + 16 * 31 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 23U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 41U; - src = *(in + 16 * 34 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 50U; - src = *(in + 16 * 35 + i); - tmp |= src << 5U; - src = *(in + 16 * 36 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 13U; - src = *(in + 16 * 38 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 22U; - src = *(in + 16 * 39 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 31U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 49U; - src = *(in + 16 * 42 + i); - tmp |= src << 6U; - src = *(in + 16 * 43 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 3U; - src = *(in + 16 * 44 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 12U; - src = *(in + 16 * 45 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 21U; - src = *(in + 16 * 46 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 30U; - src = *(in + 16 * 47 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 39U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 7U; - src = *(in + 16 * 50 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 2U; - src = *(in + 16 * 51 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 11U; - src = *(in + 16 * 52 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 20U; - src = *(in + 16 * 53 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 29U; - src = *(in + 16 * 54 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 38U; - src = *(in + 16 * 55 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 47U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 1U; - src = *(in + 16 * 58 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 10U; - src = *(in + 16 * 59 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 19U; - src = *(in + 16 * 60 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 28U; - src = *(in + 16 * 61 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 37U; - src = *(in + 16 * 62 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 46U; - src = *(in + 16 * 63 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out -= 864; - } -} -void pack_56bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 8U; - src = *(in + 16 * 2 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 16U; - src = *(in + 16 * 3 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 24U; - src = *(in + 16 * 4 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 32U; - src = *(in + 16 * 5 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 40U; - src = *(in + 16 * 6 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 48U; - src = *(in + 16 * 7 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - src = *(in + 16 * 9 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 8U; - src = *(in + 16 * 10 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 16U; - src = *(in + 16 * 11 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 24U; - src = *(in + 16 * 12 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 32U; - src = *(in + 16 * 13 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 40U; - src = *(in + 16 * 14 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 48U; - src = *(in + 16 * 15 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 8U; - src = *(in + 16 * 18 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 16U; - src = *(in + 16 * 19 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 24U; - src = *(in + 16 * 20 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 32U; - src = *(in + 16 * 21 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 40U; - src = *(in + 16 * 22 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 48U; - src = *(in + 16 * 23 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - src = *(in + 16 * 25 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 8U; - src = *(in + 16 * 26 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 16U; - src = *(in + 16 * 27 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 24U; - src = *(in + 16 * 28 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 32U; - src = *(in + 16 * 29 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 40U; - src = *(in + 16 * 30 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 48U; - src = *(in + 16 * 31 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 8U; - src = *(in + 16 * 34 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 16U; - src = *(in + 16 * 35 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 24U; - src = *(in + 16 * 36 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 32U; - src = *(in + 16 * 37 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 40U; - src = *(in + 16 * 38 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 48U; - src = *(in + 16 * 39 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - src = *(in + 16 * 41 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 8U; - src = *(in + 16 * 42 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 16U; - src = *(in + 16 * 43 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 24U; - src = *(in + 16 * 44 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 32U; - src = *(in + 16 * 45 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 40U; - src = *(in + 16 * 46 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 48U; - src = *(in + 16 * 47 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 8U; - src = *(in + 16 * 50 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 16U; - src = *(in + 16 * 51 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 24U; - src = *(in + 16 * 52 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 32U; - src = *(in + 16 * 53 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 40U; - src = *(in + 16 * 54 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 48U; - src = *(in + 16 * 55 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - src = *(in + 16 * 57 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 8U; - src = *(in + 16 * 58 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 16U; - src = *(in + 16 * 59 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 24U; - src = *(in + 16 * 60 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 32U; - src = *(in + 16 * 61 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 40U; - src = *(in + 16 * 62 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 48U; - src = *(in + 16 * 63 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out -= 880; - } -} -void pack_57bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 7U; - src = *(in + 16 * 2 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 14U; - src = *(in + 16 * 3 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 21U; - src = *(in + 16 * 4 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 28U; - src = *(in + 16 * 5 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 35U; - src = *(in + 16 * 6 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 42U; - src = *(in + 16 * 7 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 49U; - src = *(in + 16 * 8 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 56U; - src = *(in + 16 * 9 + i); - tmp |= src << 1U; - src = *(in + 16 * 10 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 6U; - src = *(in + 16 * 11 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 13U; - src = *(in + 16 * 12 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 20U; - src = *(in + 16 * 13 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 27U; - src = *(in + 16 * 14 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 34U; - src = *(in + 16 * 15 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 41U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 55U; - src = *(in + 16 * 18 + i); - tmp |= src << 2U; - src = *(in + 16 * 19 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 5U; - src = *(in + 16 * 20 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 12U; - src = *(in + 16 * 21 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 19U; - src = *(in + 16 * 22 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 26U; - src = *(in + 16 * 23 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 33U; - src = *(in + 16 * 24 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 40U; - src = *(in + 16 * 25 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 47U; - src = *(in + 16 * 26 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 54U; - src = *(in + 16 * 27 + i); - tmp |= src << 3U; - src = *(in + 16 * 28 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 4U; - src = *(in + 16 * 29 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 11U; - src = *(in + 16 * 30 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 18U; - src = *(in + 16 * 31 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 25U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 39U; - src = *(in + 16 * 34 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 46U; - src = *(in + 16 * 35 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 53U; - src = *(in + 16 * 36 + i); - tmp |= src << 4U; - src = *(in + 16 * 37 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 3U; - src = *(in + 16 * 38 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 10U; - src = *(in + 16 * 39 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 17U; - src = *(in + 16 * 40 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 24U; - src = *(in + 16 * 41 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 31U; - src = *(in + 16 * 42 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 38U; - src = *(in + 16 * 43 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 45U; - src = *(in + 16 * 44 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 52U; - src = *(in + 16 * 45 + i); - tmp |= src << 5U; - src = *(in + 16 * 46 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 2U; - src = *(in + 16 * 47 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 9U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 23U; - src = *(in + 16 * 50 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 30U; - src = *(in + 16 * 51 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 37U; - src = *(in + 16 * 52 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 44U; - src = *(in + 16 * 53 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 51U; - src = *(in + 16 * 54 + i); - tmp |= src << 6U; - src = *(in + 16 * 55 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 1U; - src = *(in + 16 * 56 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 8U; - src = *(in + 16 * 57 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 15U; - src = *(in + 16 * 58 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 22U; - src = *(in + 16 * 59 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 29U; - src = *(in + 16 * 60 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 36U; - src = *(in + 16 * 61 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 43U; - src = *(in + 16 * 62 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 50U; - src = *(in + 16 * 63 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out -= 896; - } -} -void pack_58bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 6U; - src = *(in + 16 * 2 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 12U; - src = *(in + 16 * 3 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 18U; - src = *(in + 16 * 4 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 24U; - src = *(in + 16 * 5 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 30U; - src = *(in + 16 * 6 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 36U; - src = *(in + 16 * 7 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 42U; - src = *(in + 16 * 8 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 48U; - src = *(in + 16 * 9 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 54U; - src = *(in + 16 * 10 + i); - tmp |= src << 4U; - src = *(in + 16 * 11 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 2U; - src = *(in + 16 * 12 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 8U; - src = *(in + 16 * 13 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 14U; - src = *(in + 16 * 14 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 20U; - src = *(in + 16 * 15 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 26U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 38U; - src = *(in + 16 * 18 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 44U; - src = *(in + 16 * 19 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 50U; - src = *(in + 16 * 20 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 56U; - src = *(in + 16 * 21 + i); - tmp |= src << 2U; - src = *(in + 16 * 22 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 4U; - src = *(in + 16 * 23 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 10U; - src = *(in + 16 * 24 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 16U; - src = *(in + 16 * 25 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 22U; - src = *(in + 16 * 26 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 28U; - src = *(in + 16 * 27 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 34U; - src = *(in + 16 * 28 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 40U; - src = *(in + 16 * 29 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 46U; - src = *(in + 16 * 30 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 52U; - src = *(in + 16 * 31 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 6U; - src = *(in + 16 * 34 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 12U; - src = *(in + 16 * 35 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 18U; - src = *(in + 16 * 36 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 24U; - src = *(in + 16 * 37 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 30U; - src = *(in + 16 * 38 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 36U; - src = *(in + 16 * 39 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 42U; - src = *(in + 16 * 40 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 48U; - src = *(in + 16 * 41 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 54U; - src = *(in + 16 * 42 + i); - tmp |= src << 4U; - src = *(in + 16 * 43 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 2U; - src = *(in + 16 * 44 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 8U; - src = *(in + 16 * 45 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 14U; - src = *(in + 16 * 46 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 20U; - src = *(in + 16 * 47 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 26U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 38U; - src = *(in + 16 * 50 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 44U; - src = *(in + 16 * 51 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 50U; - src = *(in + 16 * 52 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 56U; - src = *(in + 16 * 53 + i); - tmp |= src << 2U; - src = *(in + 16 * 54 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 4U; - src = *(in + 16 * 55 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 10U; - src = *(in + 16 * 56 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 16U; - src = *(in + 16 * 57 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 22U; - src = *(in + 16 * 58 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 28U; - src = *(in + 16 * 59 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 34U; - src = *(in + 16 * 60 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 40U; - src = *(in + 16 * 61 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 46U; - src = *(in + 16 * 62 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 52U; - src = *(in + 16 * 63 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out -= 912; - } -} -void pack_59bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 5U; - src = *(in + 16 * 2 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 10U; - src = *(in + 16 * 3 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 15U; - src = *(in + 16 * 4 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 20U; - src = *(in + 16 * 5 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 25U; - src = *(in + 16 * 6 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 30U; - src = *(in + 16 * 7 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 35U; - src = *(in + 16 * 8 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 40U; - src = *(in + 16 * 9 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 45U; - src = *(in + 16 * 10 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 50U; - src = *(in + 16 * 11 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 55U; - src = *(in + 16 * 12 + i); - tmp |= src << 4U; - src = *(in + 16 * 13 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 1U; - src = *(in + 16 * 14 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 6U; - src = *(in + 16 * 15 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 11U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 21U; - src = *(in + 16 * 18 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 26U; - src = *(in + 16 * 19 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 31U; - src = *(in + 16 * 20 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 36U; - src = *(in + 16 * 21 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 41U; - src = *(in + 16 * 22 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 46U; - src = *(in + 16 * 23 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 51U; - src = *(in + 16 * 24 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 56U; - src = *(in + 16 * 25 + i); - tmp |= src << 3U; - src = *(in + 16 * 26 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 2U; - src = *(in + 16 * 27 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 7U; - src = *(in + 16 * 28 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 12U; - src = *(in + 16 * 29 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 17U; - src = *(in + 16 * 30 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 22U; - src = *(in + 16 * 31 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 27U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 37U; - src = *(in + 16 * 34 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 42U; - src = *(in + 16 * 35 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 47U; - src = *(in + 16 * 36 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 52U; - src = *(in + 16 * 37 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 57U; - src = *(in + 16 * 38 + i); - tmp |= src << 2U; - src = *(in + 16 * 39 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 3U; - src = *(in + 16 * 40 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 8U; - src = *(in + 16 * 41 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 13U; - src = *(in + 16 * 42 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 18U; - src = *(in + 16 * 43 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 23U; - src = *(in + 16 * 44 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 28U; - src = *(in + 16 * 45 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 33U; - src = *(in + 16 * 46 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 38U; - src = *(in + 16 * 47 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 43U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 53U; - src = *(in + 16 * 50 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 58U; - src = *(in + 16 * 51 + i); - tmp |= src << 1U; - src = *(in + 16 * 52 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 4U; - src = *(in + 16 * 53 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 9U; - src = *(in + 16 * 54 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 14U; - src = *(in + 16 * 55 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 19U; - src = *(in + 16 * 56 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 24U; - src = *(in + 16 * 57 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 29U; - src = *(in + 16 * 58 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 34U; - src = *(in + 16 * 59 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 39U; - src = *(in + 16 * 60 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 44U; - src = *(in + 16 * 61 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 49U; - src = *(in + 16 * 62 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 54U; - src = *(in + 16 * 63 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out -= 928; - } -} -void pack_60bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 4U; - src = *(in + 16 * 2 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 8U; - src = *(in + 16 * 3 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 12U; - src = *(in + 16 * 4 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 16U; - src = *(in + 16 * 5 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 20U; - src = *(in + 16 * 6 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 24U; - src = *(in + 16 * 7 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 28U; - src = *(in + 16 * 8 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 32U; - src = *(in + 16 * 9 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 36U; - src = *(in + 16 * 10 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 40U; - src = *(in + 16 * 11 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 44U; - src = *(in + 16 * 12 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 48U; - src = *(in + 16 * 13 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 52U; - src = *(in + 16 * 14 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 56U; - src = *(in + 16 * 15 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - src = *(in + 16 * 17 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 4U; - src = *(in + 16 * 18 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 8U; - src = *(in + 16 * 19 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 12U; - src = *(in + 16 * 20 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 16U; - src = *(in + 16 * 21 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 20U; - src = *(in + 16 * 22 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 24U; - src = *(in + 16 * 23 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 28U; - src = *(in + 16 * 24 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 32U; - src = *(in + 16 * 25 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 36U; - src = *(in + 16 * 26 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 40U; - src = *(in + 16 * 27 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 44U; - src = *(in + 16 * 28 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 48U; - src = *(in + 16 * 29 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 52U; - src = *(in + 16 * 30 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 56U; - src = *(in + 16 * 31 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 4U; - src = *(in + 16 * 34 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 8U; - src = *(in + 16 * 35 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 12U; - src = *(in + 16 * 36 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 16U; - src = *(in + 16 * 37 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 20U; - src = *(in + 16 * 38 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 24U; - src = *(in + 16 * 39 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 28U; - src = *(in + 16 * 40 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 32U; - src = *(in + 16 * 41 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 36U; - src = *(in + 16 * 42 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 40U; - src = *(in + 16 * 43 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 44U; - src = *(in + 16 * 44 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 48U; - src = *(in + 16 * 45 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 52U; - src = *(in + 16 * 46 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 56U; - src = *(in + 16 * 47 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - src = *(in + 16 * 49 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 4U; - src = *(in + 16 * 50 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 8U; - src = *(in + 16 * 51 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 12U; - src = *(in + 16 * 52 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 16U; - src = *(in + 16 * 53 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 20U; - src = *(in + 16 * 54 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 24U; - src = *(in + 16 * 55 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 28U; - src = *(in + 16 * 56 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 32U; - src = *(in + 16 * 57 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 36U; - src = *(in + 16 * 58 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 40U; - src = *(in + 16 * 59 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 44U; - src = *(in + 16 * 60 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 48U; - src = *(in + 16 * 61 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 52U; - src = *(in + 16 * 62 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 56U; - src = *(in + 16 * 63 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out -= 944; - } -} -void pack_61bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 3U; - src = *(in + 16 * 2 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 6U; - src = *(in + 16 * 3 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 9U; - src = *(in + 16 * 4 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 12U; - src = *(in + 16 * 5 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 15U; - src = *(in + 16 * 6 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 18U; - src = *(in + 16 * 7 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 21U; - src = *(in + 16 * 8 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 24U; - src = *(in + 16 * 9 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 27U; - src = *(in + 16 * 10 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 30U; - src = *(in + 16 * 11 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 33U; - src = *(in + 16 * 12 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 36U; - src = *(in + 16 * 13 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 39U; - src = *(in + 16 * 14 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 42U; - src = *(in + 16 * 15 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 45U; - src = *(in + 16 * 16 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 48U; - src = *(in + 16 * 17 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 51U; - src = *(in + 16 * 18 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 54U; - src = *(in + 16 * 19 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 57U; - src = *(in + 16 * 20 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 60U; - src = *(in + 16 * 21 + i); - tmp |= src << 1U; - src = *(in + 16 * 22 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 2U; - src = *(in + 16 * 23 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 5U; - src = *(in + 16 * 24 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 8U; - src = *(in + 16 * 25 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 11U; - src = *(in + 16 * 26 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 14U; - src = *(in + 16 * 27 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 17U; - src = *(in + 16 * 28 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 20U; - src = *(in + 16 * 29 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 23U; - src = *(in + 16 * 30 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 26U; - src = *(in + 16 * 31 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 29U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 35U; - src = *(in + 16 * 34 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 38U; - src = *(in + 16 * 35 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 41U; - src = *(in + 16 * 36 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 44U; - src = *(in + 16 * 37 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 47U; - src = *(in + 16 * 38 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 50U; - src = *(in + 16 * 39 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 53U; - src = *(in + 16 * 40 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 56U; - src = *(in + 16 * 41 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 59U; - src = *(in + 16 * 42 + i); - tmp |= src << 2U; - src = *(in + 16 * 43 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 1U; - src = *(in + 16 * 44 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 4U; - src = *(in + 16 * 45 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 7U; - src = *(in + 16 * 46 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 10U; - src = *(in + 16 * 47 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 13U; - src = *(in + 16 * 48 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 16U; - src = *(in + 16 * 49 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 19U; - src = *(in + 16 * 50 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 22U; - src = *(in + 16 * 51 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 25U; - src = *(in + 16 * 52 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 28U; - src = *(in + 16 * 53 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 31U; - src = *(in + 16 * 54 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 34U; - src = *(in + 16 * 55 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 37U; - src = *(in + 16 * 56 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 40U; - src = *(in + 16 * 57 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 43U; - src = *(in + 16 * 58 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 46U; - src = *(in + 16 * 59 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 49U; - src = *(in + 16 * 60 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 52U; - src = *(in + 16 * 61 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 55U; - src = *(in + 16 * 62 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 58U; - src = *(in + 16 * 63 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out -= 960; - } -} -void pack_62bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 2U; - src = *(in + 16 * 2 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 4U; - src = *(in + 16 * 3 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 6U; - src = *(in + 16 * 4 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 8U; - src = *(in + 16 * 5 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 10U; - src = *(in + 16 * 6 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 12U; - src = *(in + 16 * 7 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 14U; - src = *(in + 16 * 8 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 16U; - src = *(in + 16 * 9 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 18U; - src = *(in + 16 * 10 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 20U; - src = *(in + 16 * 11 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 22U; - src = *(in + 16 * 12 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 24U; - src = *(in + 16 * 13 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 26U; - src = *(in + 16 * 14 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 28U; - src = *(in + 16 * 15 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 30U; - src = *(in + 16 * 16 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 32U; - src = *(in + 16 * 17 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 34U; - src = *(in + 16 * 18 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 36U; - src = *(in + 16 * 19 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 38U; - src = *(in + 16 * 20 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 40U; - src = *(in + 16 * 21 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 42U; - src = *(in + 16 * 22 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 44U; - src = *(in + 16 * 23 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 46U; - src = *(in + 16 * 24 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 48U; - src = *(in + 16 * 25 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 50U; - src = *(in + 16 * 26 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 52U; - src = *(in + 16 * 27 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 54U; - src = *(in + 16 * 28 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 56U; - src = *(in + 16 * 29 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 58U; - src = *(in + 16 * 30 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 60U; - src = *(in + 16 * 31 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - src = *(in + 16 * 33 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 2U; - src = *(in + 16 * 34 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 4U; - src = *(in + 16 * 35 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 6U; - src = *(in + 16 * 36 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 8U; - src = *(in + 16 * 37 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 10U; - src = *(in + 16 * 38 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 12U; - src = *(in + 16 * 39 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 14U; - src = *(in + 16 * 40 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 16U; - src = *(in + 16 * 41 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 18U; - src = *(in + 16 * 42 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 20U; - src = *(in + 16 * 43 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 22U; - src = *(in + 16 * 44 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 24U; - src = *(in + 16 * 45 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 26U; - src = *(in + 16 * 46 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 28U; - src = *(in + 16 * 47 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 30U; - src = *(in + 16 * 48 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 32U; - src = *(in + 16 * 49 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 34U; - src = *(in + 16 * 50 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 36U; - src = *(in + 16 * 51 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 38U; - src = *(in + 16 * 52 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 40U; - src = *(in + 16 * 53 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 42U; - src = *(in + 16 * 54 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 44U; - src = *(in + 16 * 55 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 46U; - src = *(in + 16 * 56 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 48U; - src = *(in + 16 * 57 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 50U; - src = *(in + 16 * 58 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 52U; - src = *(in + 16 * 59 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 54U; - src = *(in + 16 * 60 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 56U; - src = *(in + 16 * 61 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 58U; - src = *(in + 16 * 62 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 60U; - src = *(in + 16 * 63 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out -= 976; - } -} -void pack_63bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - src = *(in + 16 * 1 + i); - tmp |= src << 63U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src >> 1U; - src = *(in + 16 * 2 + i); - tmp |= src << 62U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src >> 2U; - src = *(in + 16 * 3 + i); - tmp |= src << 61U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src >> 3U; - src = *(in + 16 * 4 + i); - tmp |= src << 60U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src >> 4U; - src = *(in + 16 * 5 + i); - tmp |= src << 59U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src >> 5U; - src = *(in + 16 * 6 + i); - tmp |= src << 58U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src >> 6U; - src = *(in + 16 * 7 + i); - tmp |= src << 57U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src >> 7U; - src = *(in + 16 * 8 + i); - tmp |= src << 56U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src >> 8U; - src = *(in + 16 * 9 + i); - tmp |= src << 55U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src >> 9U; - src = *(in + 16 * 10 + i); - tmp |= src << 54U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src >> 10U; - src = *(in + 16 * 11 + i); - tmp |= src << 53U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src >> 11U; - src = *(in + 16 * 12 + i); - tmp |= src << 52U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src >> 12U; - src = *(in + 16 * 13 + i); - tmp |= src << 51U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src >> 13U; - src = *(in + 16 * 14 + i); - tmp |= src << 50U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src >> 14U; - src = *(in + 16 * 15 + i); - tmp |= src << 49U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src >> 15U; - src = *(in + 16 * 16 + i); - tmp |= src << 48U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src >> 16U; - src = *(in + 16 * 17 + i); - tmp |= src << 47U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src >> 17U; - src = *(in + 16 * 18 + i); - tmp |= src << 46U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src >> 18U; - src = *(in + 16 * 19 + i); - tmp |= src << 45U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src >> 19U; - src = *(in + 16 * 20 + i); - tmp |= src << 44U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src >> 20U; - src = *(in + 16 * 21 + i); - tmp |= src << 43U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src >> 21U; - src = *(in + 16 * 22 + i); - tmp |= src << 42U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src >> 22U; - src = *(in + 16 * 23 + i); - tmp |= src << 41U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src >> 23U; - src = *(in + 16 * 24 + i); - tmp |= src << 40U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src >> 24U; - src = *(in + 16 * 25 + i); - tmp |= src << 39U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src >> 25U; - src = *(in + 16 * 26 + i); - tmp |= src << 38U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src >> 26U; - src = *(in + 16 * 27 + i); - tmp |= src << 37U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src >> 27U; - src = *(in + 16 * 28 + i); - tmp |= src << 36U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src >> 28U; - src = *(in + 16 * 29 + i); - tmp |= src << 35U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src >> 29U; - src = *(in + 16 * 30 + i); - tmp |= src << 34U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src >> 30U; - src = *(in + 16 * 31 + i); - tmp |= src << 33U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src >> 31U; - src = *(in + 16 * 32 + i); - tmp |= src << 32U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src >> 32U; - src = *(in + 16 * 33 + i); - tmp |= src << 31U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src >> 33U; - src = *(in + 16 * 34 + i); - tmp |= src << 30U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src >> 34U; - src = *(in + 16 * 35 + i); - tmp |= src << 29U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src >> 35U; - src = *(in + 16 * 36 + i); - tmp |= src << 28U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src >> 36U; - src = *(in + 16 * 37 + i); - tmp |= src << 27U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src >> 37U; - src = *(in + 16 * 38 + i); - tmp |= src << 26U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src >> 38U; - src = *(in + 16 * 39 + i); - tmp |= src << 25U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src >> 39U; - src = *(in + 16 * 40 + i); - tmp |= src << 24U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src >> 40U; - src = *(in + 16 * 41 + i); - tmp |= src << 23U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src >> 41U; - src = *(in + 16 * 42 + i); - tmp |= src << 22U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src >> 42U; - src = *(in + 16 * 43 + i); - tmp |= src << 21U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src >> 43U; - src = *(in + 16 * 44 + i); - tmp |= src << 20U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src >> 44U; - src = *(in + 16 * 45 + i); - tmp |= src << 19U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src >> 45U; - src = *(in + 16 * 46 + i); - tmp |= src << 18U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src >> 46U; - src = *(in + 16 * 47 + i); - tmp |= src << 17U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src >> 47U; - src = *(in + 16 * 48 + i); - tmp |= src << 16U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src >> 48U; - src = *(in + 16 * 49 + i); - tmp |= src << 15U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src >> 49U; - src = *(in + 16 * 50 + i); - tmp |= src << 14U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src >> 50U; - src = *(in + 16 * 51 + i); - tmp |= src << 13U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src >> 51U; - src = *(in + 16 * 52 + i); - tmp |= src << 12U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src >> 52U; - src = *(in + 16 * 53 + i); - tmp |= src << 11U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src >> 53U; - src = *(in + 16 * 54 + i); - tmp |= src << 10U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src >> 54U; - src = *(in + 16 * 55 + i); - tmp |= src << 9U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src >> 55U; - src = *(in + 16 * 56 + i); - tmp |= src << 8U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src >> 56U; - src = *(in + 16 * 57 + i); - tmp |= src << 7U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src >> 57U; - src = *(in + 16 * 58 + i); - tmp |= src << 6U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src >> 58U; - src = *(in + 16 * 59 + i); - tmp |= src << 5U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src >> 59U; - src = *(in + 16 * 60 + i); - tmp |= src << 4U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src >> 60U; - src = *(in + 16 * 61 + i); - tmp |= src << 3U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src >> 61U; - src = *(in + 16 * 62 + i); - tmp |= src << 2U; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src >> 62U; - src = *(in + 16 * 63 + i); - tmp |= src << 1U; - *(out + i) = tmp; - out -= 992; - } -} -void pack_64bit_64simdWidth(uint64_t* __restrict in, uint64_t* __restrict out) { - uint64_t tmp = 0U; - uint64_t src; - for (int i = 0; i < 16; i++) { - src = *(in + 16 * 0 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 1 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 2 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 3 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 4 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 5 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 6 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 7 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 8 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 9 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 10 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 11 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 12 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 13 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 14 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 15 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 16 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 17 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 18 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 19 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 20 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 21 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 22 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 23 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 24 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 25 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 26 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 27 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 28 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 29 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 30 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 31 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 32 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 33 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 34 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 35 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 36 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 37 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 38 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 39 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 40 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 41 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 42 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 43 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 44 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 45 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 46 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 47 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 48 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 49 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 50 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 51 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 52 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 53 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 54 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 55 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 56 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 57 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 58 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 59 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 60 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 61 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 62 + i); - tmp = src; - *(out + i) = tmp; - out += 16; - src = *(in + 16 * 63 + i); - tmp = src; - *(out + i) = tmp; - out -= 1008; - } -} - -void Compress08(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - uint8_t* __restrict in = 0; - uint8_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - pack_1bit_8simdWidth(in, out); - break; - case 2: - pack_2bit_8simdWidth(in, out); - break; - case 3: - pack_3bit_8simdWidth(in, out); - break; - case 4: - pack_4bit_8simdWidth(in, out); - break; - case 5: - pack_5bit_8simdWidth(in, out); - break; - case 6: - pack_6bit_8simdWidth(in, out); - break; - case 7: - pack_7bit_8simdWidth(in, out); - break; - case 8: - pack_8bit_8simdWidth(in, out); - break; - } -} - -void Compress16(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - uint16_t* __restrict in = 0; - uint16_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - pack_1bit_16simdWidth(in, out); - break; - case 2: - pack_2bit_16simdWidth(in, out); - break; - case 3: - pack_3bit_16simdWidth(in, out); - break; - case 4: - pack_4bit_16simdWidth(in, out); - break; - case 5: - pack_5bit_16simdWidth(in, out); - break; - case 6: - pack_6bit_16simdWidth(in, out); - break; - case 7: - pack_7bit_16simdWidth(in, out); - break; - case 8: - pack_8bit_16simdWidth(in, out); - break; - case 9: - pack_9bit_16simdWidth(in, out); - break; - case 10: - pack_10bit_16simdWidth(in, out); - break; - case 11: - pack_11bit_16simdWidth(in, out); - break; - case 12: - pack_12bit_16simdWidth(in, out); - break; - case 13: - pack_13bit_16simdWidth(in, out); - break; - case 14: - pack_14bit_16simdWidth(in, out); - break; - case 15: - pack_15bit_16simdWidth(in, out); - break; - case 16: - pack_16bit_16simdWidth(in, out); - break; - } -} -void Compress32(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - uint32_t* __restrict in = 0; - uint32_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - pack_1bit_32simdWidth(in, out); - break; - case 2: - pack_2bit_32simdWidth(in, out); - break; - case 3: - pack_3bit_32simdWidth(in, out); - break; - case 4: - pack_4bit_32simdWidth(in, out); - break; - case 5: - pack_5bit_32simdWidth(in, out); - break; - case 6: - pack_6bit_32simdWidth(in, out); - break; - case 7: - pack_7bit_32simdWidth(in, out); - break; - case 8: - pack_8bit_32simdWidth(in, out); - break; - case 9: - pack_9bit_32simdWidth(in, out); - break; - case 10: - pack_10bit_32simdWidth(in, out); - break; - case 11: - pack_11bit_32simdWidth(in, out); - break; - case 12: - pack_12bit_32simdWidth(in, out); - break; - case 13: - pack_13bit_32simdWidth(in, out); - break; - case 14: - pack_14bit_32simdWidth(in, out); - break; - case 15: - pack_15bit_32simdWidth(in, out); - break; - case 16: - pack_16bit_32simdWidth(in, out); - break; - case 17: - pack_17bit_32simdWidth(in, out); - break; - case 18: - pack_18bit_32simdWidth(in, out); - break; - case 19: - pack_19bit_32simdWidth(in, out); - break; - case 20: - pack_20bit_32simdWidth(in, out); - break; - case 21: - pack_21bit_32simdWidth(in, out); - break; - case 22: - pack_22bit_32simdWidth(in, out); - break; - case 23: - pack_23bit_32simdWidth(in, out); - break; - case 24: - pack_24bit_32simdWidth(in, out); - break; - case 25: - pack_25bit_32simdWidth(in, out); - break; - case 26: - pack_26bit_32simdWidth(in, out); - break; - case 27: - pack_27bit_32simdWidth(in, out); - break; - case 28: - pack_28bit_32simdWidth(in, out); - break; - case 29: - pack_29bit_32simdWidth(in, out); - break; - case 30: - pack_30bit_32simdWidth(in, out); - break; - case 31: - pack_31bit_32simdWidth(in, out); - break; - case 32: - pack_32bit_32simdWidth(in, out); - break; - } -} - -void Compress64(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - // auto *bit_col = reinterpret_cast(state.col1); - // auto out = reinterpret_cast(state.data); - // auto in = reinterpret_cast(state.data); - - uint8_t bit_c = 0; - uint64_t* in = 0; - uint64_t* out = 0; - - switch (bit_c) { - case 0: - break; - case 1: - pack_1bit_16simdWidth(in, out); - break; - case 2: - pack_2bit_64simdWidth(in, out); - break; - case 3: - pack_3bit_64simdWidth(in, out); - break; - case 4: - pack_4bit_64simdWidth(in, out); - break; - case 5: - pack_5bit_64simdWidth(in, out); - break; - case 6: - pack_6bit_64simdWidth(in, out); - break; - case 7: - pack_7bit_64simdWidth(in, out); - break; - case 8: - pack_8bit_64simdWidth(in, out); - break; - case 9: - pack_9bit_64simdWidth(in, out); - break; - case 10: - pack_10bit_64simdWidth(in, out); - break; - case 11: - pack_11bit_64simdWidth(in, out); - break; - case 12: - pack_12bit_64simdWidth(in, out); - break; - case 13: - pack_13bit_64simdWidth(in, out); - break; - case 14: - pack_14bit_64simdWidth(in, out); - break; - case 15: - pack_15bit_64simdWidth(in, out); - break; - case 16: - pack_16bit_64simdWidth(in, out); - break; - case 17: - pack_17bit_64simdWidth(in, out); - break; - case 18: - pack_18bit_64simdWidth(in, out); - break; - case 19: - pack_19bit_64simdWidth(in, out); - break; - case 20: - pack_20bit_64simdWidth(in, out); - break; - case 21: - pack_21bit_64simdWidth(in, out); - break; - case 22: - pack_22bit_64simdWidth(in, out); - break; - case 23: - pack_23bit_64simdWidth(in, out); - break; - case 24: - pack_24bit_64simdWidth(in, out); - break; - case 25: - pack_25bit_64simdWidth(in, out); - break; - case 26: - pack_26bit_64simdWidth(in, out); - break; - case 27: - pack_27bit_64simdWidth(in, out); - break; - case 28: - pack_28bit_64simdWidth(in, out); - break; - case 29: - pack_29bit_64simdWidth(in, out); - break; - case 30: - pack_30bit_64simdWidth(in, out); - break; - case 31: - pack_31bit_64simdWidth(in, out); - break; - case 32: - pack_32bit_64simdWidth(in, out); - break; - case 33: - pack_33bit_64simdWidth(in, out); - break; - case 34: - pack_34bit_64simdWidth(in, out); - break; - case 35: - pack_35bit_64simdWidth(in, out); - break; - case 36: - pack_36bit_64simdWidth(in, out); - break; - case 37: - pack_37bit_64simdWidth(in, out); - break; - case 38: - pack_38bit_64simdWidth(in, out); - break; - case 39: - pack_39bit_64simdWidth(in, out); - break; - case 40: - pack_40bit_64simdWidth(in, out); - break; - case 41: - pack_41bit_64simdWidth(in, out); - break; - case 42: - pack_42bit_64simdWidth(in, out); - break; - case 43: - pack_43bit_64simdWidth(in, out); - break; - case 44: - pack_44bit_64simdWidth(in, out); - break; - case 45: - pack_45bit_64simdWidth(in, out); - break; - case 46: - pack_46bit_64simdWidth(in, out); - break; - case 47: - pack_47bit_64simdWidth(in, out); - break; - case 48: - pack_48bit_64simdWidth(in, out); - break; - case 49: - pack_49bit_64simdWidth(in, out); - break; - case 50: - pack_50bit_64simdWidth(in, out); - break; - case 51: - pack_51bit_64simdWidth(in, out); - break; - case 52: - pack_52bit_64simdWidth(in, out); - break; - case 53: - pack_53bit_64simdWidth(in, out); - break; - case 54: - pack_54bit_64simdWidth(in, out); - break; - case 55: - pack_55bit_64simdWidth(in, out); - break; - case 56: - pack_56bit_64simdWidth(in, out); - break; - case 57: - pack_57bit_64simdWidth(in, out); - break; - case 58: - pack_58bit_64simdWidth(in, out); - break; - case 59: - pack_59bit_64simdWidth(in, out); - break; - case 60: - pack_60bit_64simdWidth(in, out); - break; - case 61: - pack_61bit_64simdWidth(in, out); - break; - case 62: - pack_62bit_64simdWidth(in, out); - break; - case 63: - pack_63bit_64simdWidth(in, out); - break; - case 64: - pack_64bit_64simdWidth(in, out); - break; - } -} -}}; // namespace fastlanes::bitpack diff --git a/src/cor/prm/bitpack/bitpack_decompress.cpp b/src/cor/prm/bitpack/bitpack_decompress.cpp deleted file mode 100644 index 10ff755b..00000000 --- a/src/cor/prm/bitpack/bitpack_decompress.cpp +++ /dev/null @@ -1,17303 +0,0 @@ -#include "fls/cor/prm/bitpack.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t - -namespace fastlanes { namespace bitpack { -static void unpack_1bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_2bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_3bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_4bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_5bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_6bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_7bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - uint8_t tmp_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 128) + (128 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 128) + (128 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 128) + (128 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 128) + (128 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 128) + (128 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 128) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 128) + (128 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = tmp_0; - } -} -static void unpack_8bw_8ow_8crw_1uf(uint8_t* __restrict _in, uint8_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint8_t register_0; - for (int i = 0; i < 128; ++i) { - register_0 = *(in + (0 * 128) + (i * 1) + 0); - *(out + (i * 1) + (0 * 128) + (128 * 0)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 128); - *(out + (i * 1) + (0 * 128) + (128 * 1)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 256); - *(out + (i * 1) + (0 * 128) + (128 * 2)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 384); - *(out + (i * 1) + (0 * 128) + (128 * 3)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 512); - *(out + (i * 1) + (0 * 128) + (128 * 4)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 640); - *(out + (i * 1) + (0 * 128) + (128 * 5)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 768); - *(out + (i * 1) + (0 * 128) + (128 * 6)) = register_0; - register_0 = *(in + (0 * 128) + (i * 1) + 896); - *(out + (i * 1) + (0 * 128) + (128 * 7)) = register_0; - } -} -static void unpack_1bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_2bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_3bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_4bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_5bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_6bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_7bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_8bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_9bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_10bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_11bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_12bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_13bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_14bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_15bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - uint16_t tmp_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; - *(out + (i * 1) + (0 * 64) + (64 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; - *(out + (i * 1) + (0 * 64) + (64 * 2)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; - *(out + (i * 1) + (0 * 64) + (64 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; - *(out + (i * 1) + (0 * 64) + (64 * 4)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; - *(out + (i * 1) + (0 * 64) + (64 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; - *(out + (i * 1) + (0 * 64) + (64 * 6)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; - *(out + (i * 1) + (0 * 64) + (64 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; - *(out + (i * 1) + (0 * 64) + (64 * 8)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; - *(out + (i * 1) + (0 * 64) + (64 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; - *(out + (i * 1) + (0 * 64) + (64 * 10)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; - *(out + (i * 1) + (0 * 64) + (64 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; - *(out + (i * 1) + (0 * 64) + (64 * 12)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; - *(out + (i * 1) + (0 * 64) + (64 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 64) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; - *(out + (i * 1) + (0 * 64) + (64 * 14)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = tmp_0; - } -} -static void unpack_16bw_16ow_16crw_1uf(uint16_t* __restrict _in, uint16_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint16_t register_0; - for (int i = 0; i < 64; ++i) { - register_0 = *(in + (0 * 64) + (i * 1) + 0); - *(out + (i * 1) + (0 * 64) + (64 * 0)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 64); - *(out + (i * 1) + (0 * 64) + (64 * 1)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 128); - *(out + (i * 1) + (0 * 64) + (64 * 2)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 192); - *(out + (i * 1) + (0 * 64) + (64 * 3)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 256); - *(out + (i * 1) + (0 * 64) + (64 * 4)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 320); - *(out + (i * 1) + (0 * 64) + (64 * 5)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 384); - *(out + (i * 1) + (0 * 64) + (64 * 6)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 448); - *(out + (i * 1) + (0 * 64) + (64 * 7)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 512); - *(out + (i * 1) + (0 * 64) + (64 * 8)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 576); - *(out + (i * 1) + (0 * 64) + (64 * 9)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 640); - *(out + (i * 1) + (0 * 64) + (64 * 10)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 704); - *(out + (i * 1) + (0 * 64) + (64 * 11)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 768); - *(out + (i * 1) + (0 * 64) + (64 * 12)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 832); - *(out + (i * 1) + (0 * 64) + (64 * 13)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 896); - *(out + (i * 1) + (0 * 64) + (64 * 14)) = register_0; - register_0 = *(in + (0 * 64) + (i * 1) + 960); - *(out + (i * 1) + (0 * 64) + (64 * 15)) = register_0; - } -} -static void unpack_1bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_2bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_3bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_4bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_5bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_6bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_7bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_8bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_9bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_10bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_11bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_12bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_13bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_14bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_15bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_16bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_17bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_18bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_19bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_20bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_21bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_22bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_23bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_24bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_25bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_26bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_27bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_28bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_29bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_30bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_31bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - uint32_t tmp_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; - *(out + (i * 1) + (0 * 32) + (32 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; - *(out + (i * 1) + (0 * 32) + (32 * 2)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; - *(out + (i * 1) + (0 * 32) + (32 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; - *(out + (i * 1) + (0 * 32) + (32 * 4)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; - *(out + (i * 1) + (0 * 32) + (32 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; - *(out + (i * 1) + (0 * 32) + (32 * 6)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; - *(out + (i * 1) + (0 * 32) + (32 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; - *(out + (i * 1) + (0 * 32) + (32 * 8)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; - *(out + (i * 1) + (0 * 32) + (32 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; - *(out + (i * 1) + (0 * 32) + (32 * 10)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; - *(out + (i * 1) + (0 * 32) + (32 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; - *(out + (i * 1) + (0 * 32) + (32 * 12)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; - *(out + (i * 1) + (0 * 32) + (32 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; - *(out + (i * 1) + (0 * 32) + (32 * 14)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; - *(out + (i * 1) + (0 * 32) + (32 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; - *(out + (i * 1) + (0 * 32) + (32 * 16)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; - *(out + (i * 1) + (0 * 32) + (32 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; - *(out + (i * 1) + (0 * 32) + (32 * 18)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; - *(out + (i * 1) + (0 * 32) + (32 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; - *(out + (i * 1) + (0 * 32) + (32 * 20)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; - *(out + (i * 1) + (0 * 32) + (32 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; - *(out + (i * 1) + (0 * 32) + (32 * 22)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; - *(out + (i * 1) + (0 * 32) + (32 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; - *(out + (i * 1) + (0 * 32) + (32 * 24)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; - *(out + (i * 1) + (0 * 32) + (32 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; - *(out + (i * 1) + (0 * 32) + (32 * 26)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; - *(out + (i * 1) + (0 * 32) + (32 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; - *(out + (i * 1) + (0 * 32) + (32 * 28)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; - *(out + (i * 1) + (0 * 32) + (32 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 32) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; - *(out + (i * 1) + (0 * 32) + (32 * 30)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = tmp_0; - } -} -static void unpack_32bw_32ow_32crw_1uf(uint32_t* __restrict _in, uint32_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint32_t register_0; - for (int i = 0; i < 32; ++i) { - register_0 = *(in + (0 * 32) + (i * 1) + 0); - *(out + (i * 1) + (0 * 32) + (32 * 0)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 32); - *(out + (i * 1) + (0 * 32) + (32 * 1)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 64); - *(out + (i * 1) + (0 * 32) + (32 * 2)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 96); - *(out + (i * 1) + (0 * 32) + (32 * 3)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 128); - *(out + (i * 1) + (0 * 32) + (32 * 4)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 160); - *(out + (i * 1) + (0 * 32) + (32 * 5)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 192); - *(out + (i * 1) + (0 * 32) + (32 * 6)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 224); - *(out + (i * 1) + (0 * 32) + (32 * 7)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 256); - *(out + (i * 1) + (0 * 32) + (32 * 8)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 288); - *(out + (i * 1) + (0 * 32) + (32 * 9)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 320); - *(out + (i * 1) + (0 * 32) + (32 * 10)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 352); - *(out + (i * 1) + (0 * 32) + (32 * 11)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 384); - *(out + (i * 1) + (0 * 32) + (32 * 12)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 416); - *(out + (i * 1) + (0 * 32) + (32 * 13)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 448); - *(out + (i * 1) + (0 * 32) + (32 * 14)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 480); - *(out + (i * 1) + (0 * 32) + (32 * 15)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 512); - *(out + (i * 1) + (0 * 32) + (32 * 16)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 544); - *(out + (i * 1) + (0 * 32) + (32 * 17)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 576); - *(out + (i * 1) + (0 * 32) + (32 * 18)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 608); - *(out + (i * 1) + (0 * 32) + (32 * 19)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 640); - *(out + (i * 1) + (0 * 32) + (32 * 20)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 672); - *(out + (i * 1) + (0 * 32) + (32 * 21)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 704); - *(out + (i * 1) + (0 * 32) + (32 * 22)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 736); - *(out + (i * 1) + (0 * 32) + (32 * 23)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 768); - *(out + (i * 1) + (0 * 32) + (32 * 24)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 800); - *(out + (i * 1) + (0 * 32) + (32 * 25)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 832); - *(out + (i * 1) + (0 * 32) + (32 * 26)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 864); - *(out + (i * 1) + (0 * 32) + (32 * 27)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 896); - *(out + (i * 1) + (0 * 32) + (32 * 28)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 928); - *(out + (i * 1) + (0 * 32) + (32 * 29)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 960); - *(out + (i * 1) + (0 * 32) + (32 * 30)) = register_0; - register_0 = *(in + (0 * 32) + (i * 1) + 992); - *(out + (i * 1) + (0 * 32) + (32 * 31)) = register_0; - } -} -static void unpack_1bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_2bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_3bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_4bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_5bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_6bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_7bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_8bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_9bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_10bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_11bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_12bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_13bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_14bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_15bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_16bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_17bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_18bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_19bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_20bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_21bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_22bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_23bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_24bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_25bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_26bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_27bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_28bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_29bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_30bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_31bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_32bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 = (register_0) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_33bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_34bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 = (register_0) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_35bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_36bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_37bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_38bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 = (register_0) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_39bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_40bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 = (register_0) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_41bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_42bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_43bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_44bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 = (register_0) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_45bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_46bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 = (register_0) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_47bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_48bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 = (register_0) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_49bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_50bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 = (register_0) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_51bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_52bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 = (register_0) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_53bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_54bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 = (register_0) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_55bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_56bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 = (register_0) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_57bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_58bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 = (register_0) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_59bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 57) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 57; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 59) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_60bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 = (register_0) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_61bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 57; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 59) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 59) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 59; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 57) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 61) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_62bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 = (register_0) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 976); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 62) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_63bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; - uint64_t tmp_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - tmp_0 = (register_0) & ((1ULL << 63) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = tmp_0; - tmp_0 = (register_0 >> 63) & ((1ULL << 1) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 16); - tmp_0 |= ((register_0) & ((1ULL << 62) - 1)) << 1; - *(out + (i * 1) + (0 * 16) + (16 * 1)) = tmp_0; - tmp_0 = (register_0 >> 62) & ((1ULL << 2) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 32); - tmp_0 |= ((register_0) & ((1ULL << 61) - 1)) << 2; - *(out + (i * 1) + (0 * 16) + (16 * 2)) = tmp_0; - tmp_0 = (register_0 >> 61) & ((1ULL << 3) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 48); - tmp_0 |= ((register_0) & ((1ULL << 60) - 1)) << 3; - *(out + (i * 1) + (0 * 16) + (16 * 3)) = tmp_0; - tmp_0 = (register_0 >> 60) & ((1ULL << 4) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 64); - tmp_0 |= ((register_0) & ((1ULL << 59) - 1)) << 4; - *(out + (i * 1) + (0 * 16) + (16 * 4)) = tmp_0; - tmp_0 = (register_0 >> 59) & ((1ULL << 5) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 80); - tmp_0 |= ((register_0) & ((1ULL << 58) - 1)) << 5; - *(out + (i * 1) + (0 * 16) + (16 * 5)) = tmp_0; - tmp_0 = (register_0 >> 58) & ((1ULL << 6) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 96); - tmp_0 |= ((register_0) & ((1ULL << 57) - 1)) << 6; - *(out + (i * 1) + (0 * 16) + (16 * 6)) = tmp_0; - tmp_0 = (register_0 >> 57) & ((1ULL << 7) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 112); - tmp_0 |= ((register_0) & ((1ULL << 56) - 1)) << 7; - *(out + (i * 1) + (0 * 16) + (16 * 7)) = tmp_0; - tmp_0 = (register_0 >> 56) & ((1ULL << 8) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 128); - tmp_0 |= ((register_0) & ((1ULL << 55) - 1)) << 8; - *(out + (i * 1) + (0 * 16) + (16 * 8)) = tmp_0; - tmp_0 = (register_0 >> 55) & ((1ULL << 9) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 144); - tmp_0 |= ((register_0) & ((1ULL << 54) - 1)) << 9; - *(out + (i * 1) + (0 * 16) + (16 * 9)) = tmp_0; - tmp_0 = (register_0 >> 54) & ((1ULL << 10) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 160); - tmp_0 |= ((register_0) & ((1ULL << 53) - 1)) << 10; - *(out + (i * 1) + (0 * 16) + (16 * 10)) = tmp_0; - tmp_0 = (register_0 >> 53) & ((1ULL << 11) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 176); - tmp_0 |= ((register_0) & ((1ULL << 52) - 1)) << 11; - *(out + (i * 1) + (0 * 16) + (16 * 11)) = tmp_0; - tmp_0 = (register_0 >> 52) & ((1ULL << 12) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 192); - tmp_0 |= ((register_0) & ((1ULL << 51) - 1)) << 12; - *(out + (i * 1) + (0 * 16) + (16 * 12)) = tmp_0; - tmp_0 = (register_0 >> 51) & ((1ULL << 13) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 208); - tmp_0 |= ((register_0) & ((1ULL << 50) - 1)) << 13; - *(out + (i * 1) + (0 * 16) + (16 * 13)) = tmp_0; - tmp_0 = (register_0 >> 50) & ((1ULL << 14) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 224); - tmp_0 |= ((register_0) & ((1ULL << 49) - 1)) << 14; - *(out + (i * 1) + (0 * 16) + (16 * 14)) = tmp_0; - tmp_0 = (register_0 >> 49) & ((1ULL << 15) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 240); - tmp_0 |= ((register_0) & ((1ULL << 48) - 1)) << 15; - *(out + (i * 1) + (0 * 16) + (16 * 15)) = tmp_0; - tmp_0 = (register_0 >> 48) & ((1ULL << 16) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 256); - tmp_0 |= ((register_0) & ((1ULL << 47) - 1)) << 16; - *(out + (i * 1) + (0 * 16) + (16 * 16)) = tmp_0; - tmp_0 = (register_0 >> 47) & ((1ULL << 17) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 272); - tmp_0 |= ((register_0) & ((1ULL << 46) - 1)) << 17; - *(out + (i * 1) + (0 * 16) + (16 * 17)) = tmp_0; - tmp_0 = (register_0 >> 46) & ((1ULL << 18) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 288); - tmp_0 |= ((register_0) & ((1ULL << 45) - 1)) << 18; - *(out + (i * 1) + (0 * 16) + (16 * 18)) = tmp_0; - tmp_0 = (register_0 >> 45) & ((1ULL << 19) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 304); - tmp_0 |= ((register_0) & ((1ULL << 44) - 1)) << 19; - *(out + (i * 1) + (0 * 16) + (16 * 19)) = tmp_0; - tmp_0 = (register_0 >> 44) & ((1ULL << 20) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 320); - tmp_0 |= ((register_0) & ((1ULL << 43) - 1)) << 20; - *(out + (i * 1) + (0 * 16) + (16 * 20)) = tmp_0; - tmp_0 = (register_0 >> 43) & ((1ULL << 21) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 336); - tmp_0 |= ((register_0) & ((1ULL << 42) - 1)) << 21; - *(out + (i * 1) + (0 * 16) + (16 * 21)) = tmp_0; - tmp_0 = (register_0 >> 42) & ((1ULL << 22) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 352); - tmp_0 |= ((register_0) & ((1ULL << 41) - 1)) << 22; - *(out + (i * 1) + (0 * 16) + (16 * 22)) = tmp_0; - tmp_0 = (register_0 >> 41) & ((1ULL << 23) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 368); - tmp_0 |= ((register_0) & ((1ULL << 40) - 1)) << 23; - *(out + (i * 1) + (0 * 16) + (16 * 23)) = tmp_0; - tmp_0 = (register_0 >> 40) & ((1ULL << 24) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 384); - tmp_0 |= ((register_0) & ((1ULL << 39) - 1)) << 24; - *(out + (i * 1) + (0 * 16) + (16 * 24)) = tmp_0; - tmp_0 = (register_0 >> 39) & ((1ULL << 25) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 400); - tmp_0 |= ((register_0) & ((1ULL << 38) - 1)) << 25; - *(out + (i * 1) + (0 * 16) + (16 * 25)) = tmp_0; - tmp_0 = (register_0 >> 38) & ((1ULL << 26) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 416); - tmp_0 |= ((register_0) & ((1ULL << 37) - 1)) << 26; - *(out + (i * 1) + (0 * 16) + (16 * 26)) = tmp_0; - tmp_0 = (register_0 >> 37) & ((1ULL << 27) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 432); - tmp_0 |= ((register_0) & ((1ULL << 36) - 1)) << 27; - *(out + (i * 1) + (0 * 16) + (16 * 27)) = tmp_0; - tmp_0 = (register_0 >> 36) & ((1ULL << 28) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 448); - tmp_0 |= ((register_0) & ((1ULL << 35) - 1)) << 28; - *(out + (i * 1) + (0 * 16) + (16 * 28)) = tmp_0; - tmp_0 = (register_0 >> 35) & ((1ULL << 29) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 464); - tmp_0 |= ((register_0) & ((1ULL << 34) - 1)) << 29; - *(out + (i * 1) + (0 * 16) + (16 * 29)) = tmp_0; - tmp_0 = (register_0 >> 34) & ((1ULL << 30) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 480); - tmp_0 |= ((register_0) & ((1ULL << 33) - 1)) << 30; - *(out + (i * 1) + (0 * 16) + (16 * 30)) = tmp_0; - tmp_0 = (register_0 >> 33) & ((1ULL << 31) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 496); - tmp_0 |= ((register_0) & ((1ULL << 32) - 1)) << 31; - *(out + (i * 1) + (0 * 16) + (16 * 31)) = tmp_0; - tmp_0 = (register_0 >> 32) & ((1ULL << 32) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 512); - tmp_0 |= ((register_0) & ((1ULL << 31) - 1)) << 32; - *(out + (i * 1) + (0 * 16) + (16 * 32)) = tmp_0; - tmp_0 = (register_0 >> 31) & ((1ULL << 33) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 528); - tmp_0 |= ((register_0) & ((1ULL << 30) - 1)) << 33; - *(out + (i * 1) + (0 * 16) + (16 * 33)) = tmp_0; - tmp_0 = (register_0 >> 30) & ((1ULL << 34) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 544); - tmp_0 |= ((register_0) & ((1ULL << 29) - 1)) << 34; - *(out + (i * 1) + (0 * 16) + (16 * 34)) = tmp_0; - tmp_0 = (register_0 >> 29) & ((1ULL << 35) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 560); - tmp_0 |= ((register_0) & ((1ULL << 28) - 1)) << 35; - *(out + (i * 1) + (0 * 16) + (16 * 35)) = tmp_0; - tmp_0 = (register_0 >> 28) & ((1ULL << 36) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 576); - tmp_0 |= ((register_0) & ((1ULL << 27) - 1)) << 36; - *(out + (i * 1) + (0 * 16) + (16 * 36)) = tmp_0; - tmp_0 = (register_0 >> 27) & ((1ULL << 37) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 592); - tmp_0 |= ((register_0) & ((1ULL << 26) - 1)) << 37; - *(out + (i * 1) + (0 * 16) + (16 * 37)) = tmp_0; - tmp_0 = (register_0 >> 26) & ((1ULL << 38) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 608); - tmp_0 |= ((register_0) & ((1ULL << 25) - 1)) << 38; - *(out + (i * 1) + (0 * 16) + (16 * 38)) = tmp_0; - tmp_0 = (register_0 >> 25) & ((1ULL << 39) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 624); - tmp_0 |= ((register_0) & ((1ULL << 24) - 1)) << 39; - *(out + (i * 1) + (0 * 16) + (16 * 39)) = tmp_0; - tmp_0 = (register_0 >> 24) & ((1ULL << 40) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 640); - tmp_0 |= ((register_0) & ((1ULL << 23) - 1)) << 40; - *(out + (i * 1) + (0 * 16) + (16 * 40)) = tmp_0; - tmp_0 = (register_0 >> 23) & ((1ULL << 41) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 656); - tmp_0 |= ((register_0) & ((1ULL << 22) - 1)) << 41; - *(out + (i * 1) + (0 * 16) + (16 * 41)) = tmp_0; - tmp_0 = (register_0 >> 22) & ((1ULL << 42) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 672); - tmp_0 |= ((register_0) & ((1ULL << 21) - 1)) << 42; - *(out + (i * 1) + (0 * 16) + (16 * 42)) = tmp_0; - tmp_0 = (register_0 >> 21) & ((1ULL << 43) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 688); - tmp_0 |= ((register_0) & ((1ULL << 20) - 1)) << 43; - *(out + (i * 1) + (0 * 16) + (16 * 43)) = tmp_0; - tmp_0 = (register_0 >> 20) & ((1ULL << 44) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 704); - tmp_0 |= ((register_0) & ((1ULL << 19) - 1)) << 44; - *(out + (i * 1) + (0 * 16) + (16 * 44)) = tmp_0; - tmp_0 = (register_0 >> 19) & ((1ULL << 45) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 720); - tmp_0 |= ((register_0) & ((1ULL << 18) - 1)) << 45; - *(out + (i * 1) + (0 * 16) + (16 * 45)) = tmp_0; - tmp_0 = (register_0 >> 18) & ((1ULL << 46) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 736); - tmp_0 |= ((register_0) & ((1ULL << 17) - 1)) << 46; - *(out + (i * 1) + (0 * 16) + (16 * 46)) = tmp_0; - tmp_0 = (register_0 >> 17) & ((1ULL << 47) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 752); - tmp_0 |= ((register_0) & ((1ULL << 16) - 1)) << 47; - *(out + (i * 1) + (0 * 16) + (16 * 47)) = tmp_0; - tmp_0 = (register_0 >> 16) & ((1ULL << 48) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 768); - tmp_0 |= ((register_0) & ((1ULL << 15) - 1)) << 48; - *(out + (i * 1) + (0 * 16) + (16 * 48)) = tmp_0; - tmp_0 = (register_0 >> 15) & ((1ULL << 49) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 784); - tmp_0 |= ((register_0) & ((1ULL << 14) - 1)) << 49; - *(out + (i * 1) + (0 * 16) + (16 * 49)) = tmp_0; - tmp_0 = (register_0 >> 14) & ((1ULL << 50) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 800); - tmp_0 |= ((register_0) & ((1ULL << 13) - 1)) << 50; - *(out + (i * 1) + (0 * 16) + (16 * 50)) = tmp_0; - tmp_0 = (register_0 >> 13) & ((1ULL << 51) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 816); - tmp_0 |= ((register_0) & ((1ULL << 12) - 1)) << 51; - *(out + (i * 1) + (0 * 16) + (16 * 51)) = tmp_0; - tmp_0 = (register_0 >> 12) & ((1ULL << 52) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 832); - tmp_0 |= ((register_0) & ((1ULL << 11) - 1)) << 52; - *(out + (i * 1) + (0 * 16) + (16 * 52)) = tmp_0; - tmp_0 = (register_0 >> 11) & ((1ULL << 53) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 848); - tmp_0 |= ((register_0) & ((1ULL << 10) - 1)) << 53; - *(out + (i * 1) + (0 * 16) + (16 * 53)) = tmp_0; - tmp_0 = (register_0 >> 10) & ((1ULL << 54) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 864); - tmp_0 |= ((register_0) & ((1ULL << 9) - 1)) << 54; - *(out + (i * 1) + (0 * 16) + (16 * 54)) = tmp_0; - tmp_0 = (register_0 >> 9) & ((1ULL << 55) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 880); - tmp_0 |= ((register_0) & ((1ULL << 8) - 1)) << 55; - *(out + (i * 1) + (0 * 16) + (16 * 55)) = tmp_0; - tmp_0 = (register_0 >> 8) & ((1ULL << 56) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 896); - tmp_0 |= ((register_0) & ((1ULL << 7) - 1)) << 56; - *(out + (i * 1) + (0 * 16) + (16 * 56)) = tmp_0; - tmp_0 = (register_0 >> 7) & ((1ULL << 57) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 912); - tmp_0 |= ((register_0) & ((1ULL << 6) - 1)) << 57; - *(out + (i * 1) + (0 * 16) + (16 * 57)) = tmp_0; - tmp_0 = (register_0 >> 6) & ((1ULL << 58) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 928); - tmp_0 |= ((register_0) & ((1ULL << 5) - 1)) << 58; - *(out + (i * 1) + (0 * 16) + (16 * 58)) = tmp_0; - tmp_0 = (register_0 >> 5) & ((1ULL << 59) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 944); - tmp_0 |= ((register_0) & ((1ULL << 4) - 1)) << 59; - *(out + (i * 1) + (0 * 16) + (16 * 59)) = tmp_0; - tmp_0 = (register_0 >> 4) & ((1ULL << 60) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 960); - tmp_0 |= ((register_0) & ((1ULL << 3) - 1)) << 60; - *(out + (i * 1) + (0 * 16) + (16 * 60)) = tmp_0; - tmp_0 = (register_0 >> 3) & ((1ULL << 61) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 976); - tmp_0 |= ((register_0) & ((1ULL << 2) - 1)) << 61; - *(out + (i * 1) + (0 * 16) + (16 * 61)) = tmp_0; - tmp_0 = (register_0 >> 2) & ((1ULL << 62) - 1); - register_0 = *(in + (0 * 16) + (i * 1) + 992); - tmp_0 |= ((register_0) & ((1ULL << 1) - 1)) << 62; - *(out + (i * 1) + (0 * 16) + (16 * 62)) = tmp_0; - tmp_0 = (register_0 >> 1) & ((1ULL << 63) - 1); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = tmp_0; - } -} -static void unpack_64bw_64ow_64crw_1uf(uint64_t* __restrict _in, uint64_t* __restrict _out) { - auto out = reinterpret_cast(_out); - auto in = reinterpret_cast(_in); - uint64_t register_0; -#pragma clang loop vectorize(enable) - for (int i = 0; i < 16; ++i) { - register_0 = *(in + (0 * 16) + (i * 1) + 0); - *(out + (i * 1) + (0 * 16) + (16 * 0)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 16); - *(out + (i * 1) + (0 * 16) + (16 * 1)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 32); - *(out + (i * 1) + (0 * 16) + (16 * 2)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 48); - *(out + (i * 1) + (0 * 16) + (16 * 3)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 64); - *(out + (i * 1) + (0 * 16) + (16 * 4)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 80); - *(out + (i * 1) + (0 * 16) + (16 * 5)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 96); - *(out + (i * 1) + (0 * 16) + (16 * 6)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 112); - *(out + (i * 1) + (0 * 16) + (16 * 7)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 128); - *(out + (i * 1) + (0 * 16) + (16 * 8)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 144); - *(out + (i * 1) + (0 * 16) + (16 * 9)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 160); - *(out + (i * 1) + (0 * 16) + (16 * 10)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 176); - *(out + (i * 1) + (0 * 16) + (16 * 11)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 192); - *(out + (i * 1) + (0 * 16) + (16 * 12)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 208); - *(out + (i * 1) + (0 * 16) + (16 * 13)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 224); - *(out + (i * 1) + (0 * 16) + (16 * 14)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 240); - *(out + (i * 1) + (0 * 16) + (16 * 15)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 256); - *(out + (i * 1) + (0 * 16) + (16 * 16)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 272); - *(out + (i * 1) + (0 * 16) + (16 * 17)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 288); - *(out + (i * 1) + (0 * 16) + (16 * 18)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 304); - *(out + (i * 1) + (0 * 16) + (16 * 19)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 320); - *(out + (i * 1) + (0 * 16) + (16 * 20)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 336); - *(out + (i * 1) + (0 * 16) + (16 * 21)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 352); - *(out + (i * 1) + (0 * 16) + (16 * 22)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 368); - *(out + (i * 1) + (0 * 16) + (16 * 23)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 384); - *(out + (i * 1) + (0 * 16) + (16 * 24)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 400); - *(out + (i * 1) + (0 * 16) + (16 * 25)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 416); - *(out + (i * 1) + (0 * 16) + (16 * 26)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 432); - *(out + (i * 1) + (0 * 16) + (16 * 27)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 448); - *(out + (i * 1) + (0 * 16) + (16 * 28)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 464); - *(out + (i * 1) + (0 * 16) + (16 * 29)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 480); - *(out + (i * 1) + (0 * 16) + (16 * 30)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 496); - *(out + (i * 1) + (0 * 16) + (16 * 31)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 512); - *(out + (i * 1) + (0 * 16) + (16 * 32)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 528); - *(out + (i * 1) + (0 * 16) + (16 * 33)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 544); - *(out + (i * 1) + (0 * 16) + (16 * 34)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 560); - *(out + (i * 1) + (0 * 16) + (16 * 35)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 576); - *(out + (i * 1) + (0 * 16) + (16 * 36)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 592); - *(out + (i * 1) + (0 * 16) + (16 * 37)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 608); - *(out + (i * 1) + (0 * 16) + (16 * 38)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 624); - *(out + (i * 1) + (0 * 16) + (16 * 39)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 640); - *(out + (i * 1) + (0 * 16) + (16 * 40)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 656); - *(out + (i * 1) + (0 * 16) + (16 * 41)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 672); - *(out + (i * 1) + (0 * 16) + (16 * 42)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 688); - *(out + (i * 1) + (0 * 16) + (16 * 43)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 704); - *(out + (i * 1) + (0 * 16) + (16 * 44)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 720); - *(out + (i * 1) + (0 * 16) + (16 * 45)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 736); - *(out + (i * 1) + (0 * 16) + (16 * 46)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 752); - *(out + (i * 1) + (0 * 16) + (16 * 47)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 768); - *(out + (i * 1) + (0 * 16) + (16 * 48)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 784); - *(out + (i * 1) + (0 * 16) + (16 * 49)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 800); - *(out + (i * 1) + (0 * 16) + (16 * 50)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 816); - *(out + (i * 1) + (0 * 16) + (16 * 51)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 832); - *(out + (i * 1) + (0 * 16) + (16 * 52)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 848); - *(out + (i * 1) + (0 * 16) + (16 * 53)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 864); - *(out + (i * 1) + (0 * 16) + (16 * 54)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 880); - *(out + (i * 1) + (0 * 16) + (16 * 55)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 896); - *(out + (i * 1) + (0 * 16) + (16 * 56)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 912); - *(out + (i * 1) + (0 * 16) + (16 * 57)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 928); - *(out + (i * 1) + (0 * 16) + (16 * 58)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 944); - *(out + (i * 1) + (0 * 16) + (16 * 59)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 960); - *(out + (i * 1) + (0 * 16) + (16 * 60)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 976); - *(out + (i * 1) + (0 * 16) + (16 * 61)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 992); - *(out + (i * 1) + (0 * 16) + (16 * 62)) = register_0; - register_0 = *(in + (0 * 16) + (i * 1) + 1008); - *(out + (i * 1) + (0 * 16) + (16 * 63)) = register_0; - } -} - -void Decompress08(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint8_t* __restrict in = 0; - uint8_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_8ow_8crw_1uf(in, out); - break; - case 2: - unpack_2bw_8ow_8crw_1uf(in, out); - break; - case 3: - unpack_3bw_8ow_8crw_1uf(in, out); - break; - case 4: - unpack_4bw_8ow_8crw_1uf(in, out); - break; - case 5: - unpack_5bw_8ow_8crw_1uf(in, out); - break; - case 6: - unpack_6bw_8ow_8crw_1uf(in, out); - break; - case 7: - unpack_7bw_8ow_8crw_1uf(in, out); - break; - case 8: - unpack_8bw_8ow_8crw_1uf(in, out); - break; - } -} -void Decompress16(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint16_t* __restrict in = 0; - uint16_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_16ow_16crw_1uf(in, out); - break; - case 2: - unpack_2bw_16ow_16crw_1uf(in, out); - break; - case 3: - unpack_3bw_16ow_16crw_1uf(in, out); - break; - case 4: - unpack_4bw_16ow_16crw_1uf(in, out); - break; - case 5: - unpack_5bw_16ow_16crw_1uf(in, out); - break; - case 6: - unpack_6bw_16ow_16crw_1uf(in, out); - break; - case 7: - unpack_7bw_16ow_16crw_1uf(in, out); - break; - case 8: - unpack_8bw_16ow_16crw_1uf(in, out); - break; - case 9: - unpack_9bw_16ow_16crw_1uf(in, out); - break; - case 10: - unpack_10bw_16ow_16crw_1uf(in, out); - break; - case 11: - unpack_11bw_16ow_16crw_1uf(in, out); - break; - case 12: - unpack_12bw_16ow_16crw_1uf(in, out); - break; - case 13: - unpack_13bw_16ow_16crw_1uf(in, out); - break; - case 14: - unpack_14bw_16ow_16crw_1uf(in, out); - break; - case 15: - unpack_15bw_16ow_16crw_1uf(in, out); - break; - case 16: - unpack_16bw_16ow_16crw_1uf(in, out); - break; - } -} - -void Decompress32(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint32_t* __restrict in = 0; - uint32_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_32ow_32crw_1uf(in, out); - break; - case 2: - unpack_2bw_32ow_32crw_1uf(in, out); - break; - case 3: - unpack_3bw_32ow_32crw_1uf(in, out); - break; - case 4: - unpack_4bw_32ow_32crw_1uf(in, out); - break; - case 5: - unpack_5bw_32ow_32crw_1uf(in, out); - break; - case 6: - unpack_6bw_32ow_32crw_1uf(in, out); - break; - case 7: - unpack_7bw_32ow_32crw_1uf(in, out); - break; - case 8: - unpack_8bw_32ow_32crw_1uf(in, out); - break; - case 9: - unpack_9bw_32ow_32crw_1uf(in, out); - break; - case 10: - unpack_10bw_32ow_32crw_1uf(in, out); - break; - case 11: - unpack_11bw_32ow_32crw_1uf(in, out); - break; - case 12: - unpack_12bw_32ow_32crw_1uf(in, out); - break; - case 13: - unpack_13bw_32ow_32crw_1uf(in, out); - break; - case 14: - unpack_14bw_32ow_32crw_1uf(in, out); - break; - case 15: - unpack_15bw_32ow_32crw_1uf(in, out); - break; - case 16: - unpack_16bw_32ow_32crw_1uf(in, out); - break; - case 17: - unpack_17bw_32ow_32crw_1uf(in, out); - break; - case 18: - unpack_18bw_32ow_32crw_1uf(in, out); - break; - case 19: - unpack_19bw_32ow_32crw_1uf(in, out); - break; - case 20: - unpack_20bw_32ow_32crw_1uf(in, out); - break; - case 21: - unpack_21bw_32ow_32crw_1uf(in, out); - break; - case 22: - unpack_22bw_32ow_32crw_1uf(in, out); - break; - case 23: - unpack_23bw_32ow_32crw_1uf(in, out); - break; - case 24: - unpack_24bw_32ow_32crw_1uf(in, out); - break; - case 25: - unpack_25bw_32ow_32crw_1uf(in, out); - break; - case 26: - unpack_26bw_32ow_32crw_1uf(in, out); - break; - case 27: - unpack_27bw_32ow_32crw_1uf(in, out); - break; - case 28: - unpack_28bw_32ow_32crw_1uf(in, out); - break; - case 29: - unpack_29bw_32ow_32crw_1uf(in, out); - break; - case 30: - unpack_30bw_32ow_32crw_1uf(in, out); - break; - case 31: - unpack_31bw_32ow_32crw_1uf(in, out); - break; - case 32: - unpack_32bw_32ow_32crw_1uf(in, out); - break; - } -} -void Decompress64(Page& src_page, Vec& des_vec, DecompressState& de_cmpr_state) { - uint64_t* __restrict in = 0; - uint64_t* __restrict out = 0; - int b = 0; - - switch (b) { - case 1: - unpack_1bw_64ow_64crw_1uf(in, out); - break; - case 2: - unpack_2bw_64ow_64crw_1uf(in, out); - break; - case 3: - unpack_3bw_64ow_64crw_1uf(in, out); - break; - case 4: - unpack_4bw_64ow_64crw_1uf(in, out); - break; - case 5: - unpack_5bw_64ow_64crw_1uf(in, out); - break; - case 6: - unpack_6bw_64ow_64crw_1uf(in, out); - break; - case 7: - unpack_7bw_64ow_64crw_1uf(in, out); - break; - case 8: - unpack_8bw_64ow_64crw_1uf(in, out); - break; - case 9: - unpack_9bw_64ow_64crw_1uf(in, out); - break; - case 10: - unpack_10bw_64ow_64crw_1uf(in, out); - break; - case 11: - unpack_11bw_64ow_64crw_1uf(in, out); - break; - case 12: - unpack_12bw_64ow_64crw_1uf(in, out); - break; - case 13: - unpack_13bw_64ow_64crw_1uf(in, out); - break; - case 14: - unpack_14bw_64ow_64crw_1uf(in, out); - break; - case 15: - unpack_15bw_64ow_64crw_1uf(in, out); - break; - case 16: - unpack_16bw_64ow_64crw_1uf(in, out); - break; - case 17: - unpack_17bw_64ow_64crw_1uf(in, out); - break; - case 18: - unpack_18bw_64ow_64crw_1uf(in, out); - break; - case 19: - unpack_19bw_64ow_64crw_1uf(in, out); - break; - case 20: - unpack_20bw_64ow_64crw_1uf(in, out); - break; - case 21: - unpack_21bw_64ow_64crw_1uf(in, out); - break; - case 22: - unpack_22bw_64ow_64crw_1uf(in, out); - break; - case 23: - unpack_23bw_64ow_64crw_1uf(in, out); - break; - case 24: - unpack_24bw_64ow_64crw_1uf(in, out); - break; - case 25: - unpack_25bw_64ow_64crw_1uf(in, out); - break; - case 26: - unpack_26bw_64ow_64crw_1uf(in, out); - break; - case 27: - unpack_27bw_64ow_64crw_1uf(in, out); - break; - case 28: - unpack_28bw_64ow_64crw_1uf(in, out); - break; - case 29: - unpack_29bw_64ow_64crw_1uf(in, out); - break; - case 30: - unpack_30bw_64ow_64crw_1uf(in, out); - break; - case 31: - unpack_31bw_64ow_64crw_1uf(in, out); - break; - case 32: - unpack_32bw_64ow_64crw_1uf(in, out); - break; - case 33: - unpack_33bw_64ow_64crw_1uf(in, out); - break; - case 34: - unpack_34bw_64ow_64crw_1uf(in, out); - break; - case 35: - unpack_35bw_64ow_64crw_1uf(in, out); - break; - case 36: - unpack_36bw_64ow_64crw_1uf(in, out); - break; - case 37: - unpack_37bw_64ow_64crw_1uf(in, out); - break; - case 38: - unpack_38bw_64ow_64crw_1uf(in, out); - break; - case 39: - unpack_39bw_64ow_64crw_1uf(in, out); - break; - case 40: - unpack_40bw_64ow_64crw_1uf(in, out); - break; - case 41: - unpack_41bw_64ow_64crw_1uf(in, out); - break; - case 42: - unpack_42bw_64ow_64crw_1uf(in, out); - break; - case 43: - unpack_43bw_64ow_64crw_1uf(in, out); - break; - case 44: - unpack_44bw_64ow_64crw_1uf(in, out); - break; - case 45: - unpack_45bw_64ow_64crw_1uf(in, out); - break; - case 46: - unpack_46bw_64ow_64crw_1uf(in, out); - break; - case 47: - unpack_47bw_64ow_64crw_1uf(in, out); - break; - case 48: - unpack_48bw_64ow_64crw_1uf(in, out); - break; - case 49: - unpack_49bw_64ow_64crw_1uf(in, out); - break; - case 50: - unpack_50bw_64ow_64crw_1uf(in, out); - break; - case 51: - unpack_51bw_64ow_64crw_1uf(in, out); - break; - case 52: - unpack_52bw_64ow_64crw_1uf(in, out); - break; - case 53: - unpack_53bw_64ow_64crw_1uf(in, out); - break; - case 54: - unpack_54bw_64ow_64crw_1uf(in, out); - break; - case 55: - unpack_55bw_64ow_64crw_1uf(in, out); - break; - case 56: - unpack_56bw_64ow_64crw_1uf(in, out); - break; - case 57: - unpack_57bw_64ow_64crw_1uf(in, out); - break; - case 58: - unpack_58bw_64ow_64crw_1uf(in, out); - break; - case 59: - unpack_59bw_64ow_64crw_1uf(in, out); - break; - case 60: - unpack_60bw_64ow_64crw_1uf(in, out); - break; - case 61: - unpack_61bw_64ow_64crw_1uf(in, out); - break; - case 62: - unpack_62bw_64ow_64crw_1uf(in, out); - break; - case 63: - unpack_63bw_64ow_64crw_1uf(in, out); - break; - case 64: - unpack_64bw_64ow_64crw_1uf(in, out); - break; - } -} -}}; // namespace fastlanes::bitpack diff --git a/src/cor/prm/factor/CMakeLists.txt b/src/cor/prm/factor/CMakeLists.txt deleted file mode 100644 index b67390a3..00000000 --- a/src/cor/prm/factor/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_factor_prm - OBJECT - decompress.cpp - compress.cpp) - -target_compile_definitions(fls_factor_prm PRIVATE IS_SCALAR) -target_compile_options(fls_factor_prm PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_factor_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_factor_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/factor/compress.cpp b/src/cor/prm/factor/compress.cpp deleted file mode 100644 index c7a9524b..00000000 --- a/src/cor/prm/factor/compress.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include "fls/cor/prm/factor.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t -#include // for __restrict - -namespace fastlanes { namespace factor { -void Compress64(Vec& src_vec, Vec& des_vec, CompressState& compress_state) { - /* */ /* Reinterpret. */ - /* -auto *factor_col = reinterpret_cast(state.col1); -auto src_vec = reinterpret_cast(src); -auto factor = factor_col[0]; - -*/ /* Compute. */ /* - for (size_t i = 0; i < 1024; ++i) { - src_vec[i] = src_vec[i] - factor; - } - - */ /* State. */ /* -// state.col1 += 8;*/ -} -}} // namespace fastlanes::factor diff --git a/src/cor/prm/factor/decompress.cpp b/src/cor/prm/factor/decompress.cpp deleted file mode 100644 index 288ac182..00000000 --- a/src/cor/prm/factor/decompress.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include "fls/cor/prm/factor.hpp" // for pack -#include // for uint64_t, uint32_t, uint16_t, uint8_t -#include // for __restrict - -namespace fastlanes { namespace factor { -}} diff --git a/src/cor/prm/ffor/CMakeLists.txt b/src/cor/prm/ffor/CMakeLists.txt deleted file mode 100644 index b950a812..00000000 --- a/src/cor/prm/ffor/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_library(fls_ffor_prm - OBJECT - ffor_decompress.cpp - ffor_compress.cpp - ffor_analyze.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_ffor_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_ffor_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/ffor/ffor_analyze.cpp b/src/cor/prm/ffor/ffor_analyze.cpp deleted file mode 100644 index cb371b25..00000000 --- a/src/cor/prm/ffor/ffor_analyze.cpp +++ /dev/null @@ -1,202 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/mtd.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/stt/histogram.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include -#include -#include - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -template -bool ffor_prm::is_exception(Params params, T val) { - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - FLS_ASSERT_CORRECT_BW(params.bw) - FLS_ASSERT_CORRECT_N(params.exc_c) - FLS_ASSERT_CORRECT_SZ(params.size()) - // [fixme] - if (params.bw == TYPE_BIT::VALUE) { - return false; - } - - T lower_bound = static_cast(params.base); - - if (val < lower_bound) { - return true; - } - if (val - lower_bound > MAX_BIT(params.bw)) { - return true; - } - return false; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return false; - } -} - -template -Params ffor_prm::find_best_params(Histogram& histogram, n_t idx) { - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - - /* Initialize */ - Params best, next; - auto& val_vec = histogram.val_vec; - auto& rep_vec = histogram.rep_vec; - const T lower_bound = val_vec[idx]; - T upper_bound = lower_bound; - - /* best = worst */ - best = {TYPE_BIT::VALUE, 0, 0}; - - // can we do better? - for (int i = TYPE_BIT::VALUE - 1; i >= 0; --i) { - /* Compute the upperbound*/ - upper_bound = SafeUpperBound(lower_bound, i); - - /* Compute the next setting*/ - next = { - static_cast(i), - lower_bound, - static_cast(typed_patch::count_exceptions(lower_bound, upper_bound, val_vec, rep_vec))}; - - FLS_ASSERT_CORRECT_EXC_C(next.exc_c) - /* Check to see if there is a better setting. */ - if (next.size() < best.size() & next.exc_c <= CFG::PATCH::EXC_LIMIT_C) { - best = next; - } - } - - return best; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return Params(); - } -} - -template -void ffor_analyze(Vec& src_vec, Vec& des_vec, CompressState& stt) { - /* copy data into des.*/ - memcpy( - des_vec.buf_arr[stt.cur_des_buff + 1].mutable_data(), src_vec.buf_arr[stt.cur_src_buff].data(), vec_bsz()); - - auto min = std::numeric_limits::max(); - auto max = std::numeric_limits::min(); - - auto* in_p = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 1].data()); - for (size_t i {0}; i < 1024; ++i) { - if (in_p[i] < min) { - min = in_p[i]; - } - if (in_p[i] > max) { - max = in_p[i]; - } - } - - uint64_t delta = (static_cast(max) - static_cast(min)); - auto bits_per_digit = ceil(log2(delta + 1)); - uint8_t bw = bits_per_digit; - - stt.bw = bw; - std::memcpy(stt.base, &min, sizeof(T)); - // base_p[0] = min; - FLS_PLOG_KEY_VALUE("bw", std::to_string(stt.bw)) - FLS_PLOG_KEY_VALUE("min", std::to_string(min)) - FLS_PLOG_KEY_VALUE("max", std::to_string(max)) -} - -template -void ffor_patch_analyze(Vec& src_vec, Vec& des_vec, CompressState& stt) { - /* copy data into stt_buf.*/ - Params best, next; - memcpy(des_vec.buf_arr[1].mutable_data(), src_vec.buf_arr[0].data(), vec_bsz()); - - /* [op]: the tmp buf is sorted already, the minmax is the first and last element. */ - /* [op]: move to stt to reduce allocation. */ - auto histo = Histogram::cal(des_vec.buf_arr[1].mutable_data()); - auto min_max = MinMax::cal(des_vec.buf_arr[1].mutable_data()); - n_t base_c = histo->rep_vec.size(); - - best = ffor_prm::find_best_params(*histo, 0); - - /* Compute. */ - for (size_t i {1}; i < base_c; ++i) { - next = ffor_prm::find_best_params(*histo, i); - - if (next.size() < best.size()) { - best = next; - } - } - - stt.bw = best.bw; - std::memcpy(stt.base, &best.base, sizeof(T)); - FLS_ASSERT_CORRECT_BASE_UB(best.base, min_max->max) - - stt.exc_c = best.exc_c; - - /* Add exception positions . */ - uint16_t exc_c {0}; - - for (size_t i {0}; i < vec_n_tup(); ++i) { - T& val = reinterpret_cast(src_vec.buf_arr[0].mutable_data())[i]; - if (ffor_prm::is_exception(best, val)) { - stt.exc_pos_arr[exc_c++] = i; - } - } - - FLS_PLOG_KEY_VALUE("VEC", ToStr::to_str(reinterpret_cast(src_vec.buf_arr[0].mutable_data()))) - FLS_PLOG_KEY_VALUE("BW", std::to_string(stt.bw)) - FLS_PLOG_KEY_VALUE("EXC_C", ToStr::to_str(exc_c)) - FLS_PLOG_KEY_VALUE("BASE", ToStr::to_hex(stt.base, sizeof(T))); -} - -template -cmpr_fun_t ffor_prm::ResolveAnalyzeFunc() { - if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else if constexpr (std::is_same()) { - return ffor_analyze; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; - } -} - -template -cmpr_fun_t ffor_prm::ResolvePatchAnalyzeFunc() { - if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else if constexpr (std::is_same()) { - return ffor_patch_analyze; - } else { - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; - } -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(ffor_prm) -} // namespace fastlanes diff --git a/src/cor/prm/ffor/ffor_compress.cpp b/src/cor/prm/ffor/ffor_compress.cpp deleted file mode 100644 index 67cc4041..00000000 --- a/src/cor/prm/ffor/ffor_compress.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/ffor.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" - -namespace fastlanes { -template -static void ffor_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* base_p = reinterpret_cast(stt.base); - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - auto out_p = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 2].mutable_data()); - - generated::ffor::fallback::scalar::ffor(in_arr, out_p, stt.bw, base_p); - - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&stt.bw, 1); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(stt.base, sizeof(T)); - des_vec.buf_arr[stt.cur_des_buff + 2].UnsafeAdvance(ffor_prm::vec_sz(stt.bw)); - - FLS_PLOG_KEY_VALUE("VEC", ToStr::to_str(const_cast(in_arr))) - FLS_PLOG_KEY_VALUE("BW", std::to_string(stt.bw)) - FLS_PLOG_KEY_VALUE("BASE", ToStr::to_hex(stt.base, sizeof(T))); - - stt.cur_des_buff += 3; -} - -template -cmpr_fun_t ffor_prm::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } else if constexpr (std::is_same()) { - return ffor_compress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -template -n_t ffor_prm::vec_sz(uint8_t bw) { - FLS_ASSERT_CORRECT_BW(bw) - - return bw * vec_sz_1bit(); -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(ffor_prm) - -} // namespace fastlanes diff --git a/src/cor/prm/ffor/ffor_decompress.cpp b/src/cor/prm/ffor/ffor_decompress.cpp deleted file mode 100644 index bff14c4a..00000000 --- a/src/cor/prm/ffor/ffor_decompress.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/ffor_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls_gen/unffor/unffor.hpp" -#include - -namespace fastlanes { -template -static void ffor_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 2]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 1]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr - 0]) - - uint8_t bw = *src.arr[stt.cur_src_arr - 2]; - auto* base_p = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* in_p = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr]); - - generated::unffor::fallback::scalar::unffor(in_p, out_p, bw, base_p); - - FLS_PLOG_KEY_VALUE("VEC", ToStr::to_str(reinterpret_cast(out_p))) - FLS_PLOG_KEY_VALUE("BW", std::to_string(bw)) - FLS_PLOG_KEY_VALUE("BASE", ToStr::to_hex(stt.base, sizeof(T))); - - stt.cur_src_arr -= 3; - std::memcpy(stt.base, base_p, sizeof(T)); -} - -template -de_cmpr_fun_t ffor_prm::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } else if constexpr (std::is_same()) { - return ffor_decompress; - } - - FLS_ABORT("THIS IS NOT SUPPORTED!") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(ffor_prm) -} // namespace fastlanes diff --git a/src/cor/prm/mem_cpy/CMakeLists.txt b/src/cor/prm/mem_cpy/CMakeLists.txt deleted file mode 100644 index f6a68144..00000000 --- a/src/cor/prm/mem_cpy/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_mem_cpy_exp - OBJECT - mem_cpy_decompress.cpp - mem_cpy_compress.cpp) - -target_compile_definitions(fls_mem_cpy_exp PRIVATE IS_SCALAR) -target_compile_options(fls_mem_cpy_exp PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(fls_mem_cpy_exp PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_mem_cpy_exp PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/mem_cpy/mem_cpy_compress.cpp b/src/cor/prm/mem_cpy/mem_cpy_compress.cpp deleted file mode 100644 index 4bf696f3..00000000 --- a/src/cor/prm/mem_cpy/mem_cpy_compress.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_cpy_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" - -namespace fastlanes { -template -static void mem_cpy_cmpr(Vec& src_vec, Vec& des_vec, CompressState& stt) { - T* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - des_vec.buf_arr[stt.cur_des_buff].UnsafeAppend(in_arr, src_vec.buf_arr[stt.cur_src_buff].length()); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(in_arr)); - - stt.cur_des_buff += 1; - stt.cur_src_buff += 1; -} - -static void off_mem_cpy_cmpr(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = src_vec.buf_arr[stt.cur_src_buff].mutable_data(); - - /* in smart offset array the last tuple show the number of bytes. */ - uint32_t byte_c = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 1].mutable_data())[1023]; - // FLS_ASSERT_LE(byte_c, CFG::BUF::SZ) // todo[buf] - - des_vec.buf_arr[stt.cur_des_buff].UnsafeAppend(in_arr, byte_c); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(in_arr)); - - stt.cur_des_buff += 1; - stt.cur_src_buff += 1; -} - -template -cmpr_fun_t mem_cpy::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return off_mem_cpy_cmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_cmpr; - } - - FLS_ABORT("IT IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, mem_cpy::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/mem_cpy/mem_cpy_decompress.cpp b/src/cor/prm/mem_cpy/mem_cpy_decompress.cpp deleted file mode 100644 index ee85e1c7..00000000 --- a/src/cor/prm/mem_cpy/mem_cpy_decompress.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_cpy_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include - -namespace fastlanes { -template -static void mem_cpy_decmpr(PageParam src, VecParam des, DecompressState& stt) { - auto* in_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); - - std::memcpy(out_arr, in_arr, vec_bsz()); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(out_arr)); - - stt.cur_des_arr -= 1; - stt.cur_src_arr -= 1; -} - -static void off_mem_cpy_decmpr(PageParam src, VecParam des, DecompressState& stt) { - auto* in_arr = src.arr[stt.cur_src_arr - 0]; - auto* out_arr = des.arr[stt.cur_des_arr - 0]; - - /* in smart offset array the last tuple show the number of bytes. */ - uint32_t byte_c = reinterpret_cast(des.arr[stt.cur_des_arr + 1])[1023]; - // FLS_ASSERT_LE(byte_c, CFG::BUF::SZ) // todo[buf] - std::memcpy(out_arr, in_arr, byte_c); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(out_arr)); - - stt.cur_des_arr -= 1; - stt.cur_src_arr -= 1; -} - -template -de_cmpr_fun_t mem_cpy::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return off_mem_cpy_decmpr; - } else if constexpr (std::is_same_v) { - return mem_cpy_decmpr; - } - - FLS_ABORT("IT IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, mem_cpy::ResolveDecompressFunc) -} // namespace fastlanes diff --git a/src/cor/prm/mem_trs/CMakeLists.txt b/src/cor/prm/mem_trs/CMakeLists.txt deleted file mode 100644 index c6ca11d7..00000000 --- a/src/cor/prm/mem_trs/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_mem_trs_prm - OBJECT - mem_trs_decompress.cpp - mem_trs_compress.cpp) - -target_compile_definitions(fls_mem_trs_prm PRIVATE IS_SCALAR) -target_compile_options(fls_mem_trs_prm PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(fls_mem_trs_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_mem_trs_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/mem_trs/mem_trs_compress.cpp b/src/cor/prm/mem_trs/mem_trs_compress.cpp deleted file mode 100644 index 26eb4ad8..00000000 --- a/src/cor/prm/mem_trs/mem_trs_compress.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_trs.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/transpose/transpose.hpp" -#include - -namespace fastlanes { -static void Compress64(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto size = src_vec.buf_arr[stt.cur_src_buff].length(); - const uint8_t* in_p = src_vec.buf_arr[stt.cur_src_buff++].mutable_data(); - - /* Fixme: not efficient. could be easily solved by a better local buffer anf memory pool manager */ - uint8_t tmp[1024 * 8]; - generated::transpose::fallback::scalar::transpose_i(reinterpret_cast(in_p), - reinterpret_cast(tmp)); - - des_vec.buf_arr[stt.cur_des_buff++].UnsafeAppend(tmp, size); -} - -static void Compress32(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto size = src_vec.buf_arr[stt.cur_src_buff].length(); - const uint8_t* in_p = src_vec.buf_arr[stt.cur_src_buff++].mutable_data(); - - uint8_t tmp[1024 * 8]; - generated::transpose::fallback::scalar::transpose_i(reinterpret_cast(in_p), - reinterpret_cast(tmp)); - - des_vec.buf_arr[stt.cur_des_buff++].UnsafeAppend(tmp, size); -} - -template -cmpr_fun_t mem_trs::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return Compress64; - } else if constexpr (std::is_same()) { - return Compress64; - } else if constexpr (std::is_same()) { - return Compress32; - } else if constexpr (std::is_same()) { - return Compress32; - } - - FLS_ABORT("IT IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, mem_trs::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/mem_trs/mem_trs_decompress.cpp b/src/cor/prm/mem_trs/mem_trs_decompress.cpp deleted file mode 100644 index 7908450e..00000000 --- a/src/cor/prm/mem_trs/mem_trs_decompress.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/mem_trs.hpp" -#include "fls_gen/untranspose/untranspose.hpp" - -namespace fastlanes { -static void Decompress64(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr]) - - generated::untranspose::fallback::scalar::untranspose_i(reinterpret_cast(src.arr[stt.cur_src_arr++]), - reinterpret_cast(des.arr[stt.cur_des_arr++])); -} - -static void Decompress32(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - FLS_ASSERT_NOT_NULL_POINTER(src.arr[stt.cur_src_arr]) - - generated::untranspose::fallback::scalar::untranspose_i(reinterpret_cast(src.arr[stt.cur_src_arr++]), - reinterpret_cast(des.arr[stt.cur_des_arr++])); -} - -template -de_cmpr_fun_t mem_trs::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return Decompress64; - } - if constexpr (std::is_same()) { - return Decompress64; - } - if constexpr (std::is_same()) { - return Decompress32; - } - if constexpr (std::is_same()) { - return Decompress32; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, mem_trs::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/off_cpy/CMakeLists.txt b/src/cor/prm/off_cpy/CMakeLists.txt deleted file mode 100644 index 8b137891..00000000 --- a/src/cor/prm/off_cpy/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/cor/prm/off_cpy/off_cpy_compress.cpp b/src/cor/prm/off_cpy/off_cpy_compress.cpp deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cor/prm/off_cpy/off_cpy_decompress.cpp b/src/cor/prm/off_cpy/off_cpy_decompress.cpp deleted file mode 100644 index e69de29b..00000000 diff --git a/src/cor/prm/patch/CMakeLists.txt b/src/cor/prm/patch/CMakeLists.txt deleted file mode 100644 index 6139dfc8..00000000 --- a/src/cor/prm/patch/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -add_subdirectory(b_patch) -add_subdirectory(ll_patch) -add_subdirectory(s_patch) - -add_library(patch_prm - OBJECT - patch.cpp) - -target_compile_definitions(patch_prm PRIVATE IS_SCALAR) -target_compile_options(patch_prm PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/b_patch/CMakeLists.txt b/src/cor/prm/patch/b_patch/CMakeLists.txt deleted file mode 100644 index 10c8be66..00000000 --- a/src/cor/prm/patch/b_patch/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_b_patch_prm - OBJECT - b_patch_decompress.cpp - b_patch_compress.cpp) - -target_compile_definitions(fls_b_patch_prm PRIVATE IS_SCALAR) -target_compile_options(fls_b_patch_prm PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_b_patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_b_patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/b_patch/b_patch_compress.cpp b/src/cor/prm/patch/b_patch/b_patch_compress.cpp deleted file mode 100644 index ed587994..00000000 --- a/src/cor/prm/patch/b_patch/b_patch_compress.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/b_patch.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" - -namespace fastlanes { -template -static void b_patch_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - patch::b_patch_compress(stt.exc_pos_arr, stt.exc_c, stt.bitmap); - FLS_PLOG_KEY_VALUE("bitmap", ToStr::to_str(reinterpret_cast(stt.bitmap))); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.bitmap, CFG::BitMap::SZ); - - for (n_t i {0}; i < stt.exc_c; i++) { - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&in_arr[stt.exc_pos_arr[i]], sizeof(T)); - } - - stt.cur_des_buff += 2; - FLS_PLOG_KEY_VALUE("bitmap", ToStr::to_str(reinterpret_cast(stt.bitmap))); -} - -template -cmpr_fun_t b_patch::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return b_patch_compress; - } - if constexpr (std::is_same()) { - return b_patch_compress; - } - if constexpr (std::is_same()) { - return b_patch_compress; - } - if constexpr (std::is_same()) { - return b_patch_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, b_patch::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/b_patch/b_patch_decompress.cpp b/src/cor/prm/patch/b_patch/b_patch_decompress.cpp deleted file mode 100644 index 90009d71..00000000 --- a/src/cor/prm/patch/b_patch/b_patch_decompress.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/b_patch.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void b_patch_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - - auto* exc_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* bitmap_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr]); - - FLS_PLOG_KEY_VALUE("out_p", ToStr::to_str(out_p)); - - typed_patch::b_patch_decompress(out_p, bitmap_arr, exc_arr); - - stt.cur_src_arr -= 2; - - FLS_PLOG_KEY_VALUE("out_p", ToStr::to_str(out_p)); - FLS_PLOG_KEY_VALUE("bitmap", ToStr::to_str(reinterpret_cast(bitmap_arr))); -} - -template -de_cmpr_fun_t b_patch::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return b_patch_decompress; - } - if constexpr (std::is_same()) { - return b_patch_decompress; - } - if constexpr (std::is_same()) { - return b_patch_decompress; - } - if constexpr (std::is_same()) { - return b_patch_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, b_patch::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/ll_patch/CMakeLists.txt b/src/cor/prm/patch/ll_patch/CMakeLists.txt deleted file mode 100644 index d5e96458..00000000 --- a/src/cor/prm/patch/ll_patch/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_library(fls_ll_patch_prm - OBJECT - ll_patch_decompress.cpp - ll_patch_compress.cpp) - -target_compile_definitions(fls_ll_patch_prm PRIVATE IS_SCALAR) -target_compile_options(fls_ll_patch_prm PRIVATE "") - -if(ENABLE_IWYU) - set_target_properties(fls_ll_patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_ll_patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/ll_patch/ll_patch_compress.cpp b/src/cor/prm/patch/ll_patch/ll_patch_compress.cpp deleted file mode 100644 index bb61025c..00000000 --- a/src/cor/prm/patch/ll_patch/ll_patch_compress.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/ll_patch.hpp" -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -template -static void ll_patch_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - const auto* base_p = reinterpret_cast(stt.base); - - rle_idx_t new_c {0}; - rle_idx_t old_c {0}; - //[fixme] - rle_idx_t tmp[vec_n_tup()]; - switch (stt.exc_c) { - case 0: - break; - default: - T curr_pos = stt.exc_pos_arr[old_c++]; - T next_pos = stt.exc_pos_arr[old_c++]; - T max_range = 1 << stt.bw; - - FLS_ASSERT_CORRECT_POS(curr_pos) - do { - tmp[new_c++] = curr_pos; - auto con = patch::is_compulsory_exc(curr_pos, next_pos, max_range); - if (old_c <= stt.exc_c && con) { - curr_pos += max_range; - } else { - curr_pos = next_pos; - next_pos = stt.exc_pos_arr[old_c++]; - } - } while (old_c <= stt.exc_c + 1); - stt.exc_c = new_c; - } - - /* push the number of exceptions. */ - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.exc_c, sizeof(rle_idx_t)); - /* push the pos of first exception. */ - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&tmp[0], sizeof(rle_idx_t)); - - /* push the exceptions and fix the linked list. */ - for (rle_idx_t i {0}; i < stt.exc_c; ++i) { - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&in_arr[tmp[i]], sizeof(T)); - - auto gap = tmp[i + 1] - tmp[i] + (*base_p) - 1; - in_arr[tmp[i]] = gap; - } - - FLS_PLOG_KEY_VALUE("after ll_patch", ToStr::to_str(in_arr)); - stt.cur_des_buff += 2; -} - -template -cmpr_fun_t ll_patch::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return ll_patch_compress; - } - if constexpr (std::is_same()) { - return ll_patch_compress; - } - if constexpr (std::is_same()) { - return ll_patch_compress; - } - if constexpr (std::is_same()) { - return ll_patch_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, ll_patch::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/ll_patch/ll_patch_decompress.cpp b/src/cor/prm/patch/ll_patch/ll_patch_decompress.cpp deleted file mode 100644 index 87f9acfe..00000000 --- a/src/cor/prm/patch/ll_patch/ll_patch_decompress.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/ll_patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void ll_patch_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - - auto* exc_p = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* exc_pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr]); - auto base = *(reinterpret_cast(stt.base)); - rle_idx_t exc_c = exc_pos_arr[0]; - - FLS_PLOG_KEY_VALUE("exe_c", std::to_string(exc_c)); - - rle_idx_t first_pos = exc_pos_arr[1]; - FLS_PLOG_KEY_VALUE("first_pos", std::to_string(first_pos)); - - FLS_PLOG_KEY_VALUE("out_arr", ToStr::to_str(out_p)); - FLS_PLOG_KEY_VALUE("exc_arr", ToStr::to_str(exc_p, exc_c)); - FLS_PLOG_KEY_VALUE("[exe_c, exc_first_pos] ", ToStr::to_str(exc_pos_arr, 2)); - - T cur = first_pos; - T offset {0}; - for (T i {0}; i < exc_c; ++i) { - offset = out_p[cur] - base + 1; - out_p[cur] = exc_p[i]; - cur += offset; - } - - stt.cur_src_arr -= 2; - - FLS_PLOG_KEY_VALUE("out_p", ToStr::to_str(out_p)); -} - -template -de_cmpr_fun_t ll_patch::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - if constexpr (std::is_same()) { - return ll_patch_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, ll_patch::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/patch.cpp b/src/cor/prm/patch/patch.cpp deleted file mode 100644 index c427e1cd..00000000 --- a/src/cor/prm/patch/patch.cpp +++ /dev/null @@ -1,152 +0,0 @@ -#include "fls/cor/prm/patch/patch.hpp" -#include "fls/cfg/cfg.hpp" -#include "fls/cor/eng/compressor.hpp" -#include "fls/utl/bit_util.hpp" - -#pragma clang diagnostic ignored "-Wconversion" - -namespace fastlanes { -template -bool typed_patch::is_exception(T lower_bound, T upper_bound, T val) { - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - if (val <= upper_bound && val >= lower_bound) { - return false; - } - return true; - } else { - FLS_ABORT("this is not supported!") - return false; - } -} - -void patch::b_patch_compress(rle_idx_t arr[], rle_idx_t c, bitmap_t bitmap) { - for (n_t i {0}; i < CFG::BitMap::UNIT_C; ++i) { - bitmap[i] = 0; - } - - for (auto i {0}; i < c; ++i) { - rle_idx_t unit_num = arr[i] / CFG::BitMap::UNIT_BIT; // Which unit? - rle_idx_t relative_idx = arr[i] % CFG::BitMap::UNIT_BIT; // Where in the unit? - bit::set(bitmap[unit_num], relative_idx); - } -} - -bool patch::is_compulsory_exc(rle_idx_t curr, rle_idx_t next, uint64_t range) { - FLS_ASSERT_CORRECT_RANGE(range) - - return static_cast(next - curr) > range; -} - -template -n_t typed_patch::count_exceptions(const T lower_bound, - const T upper_bound, - const std::vector& val_vec, - const std::vector& rep_vec) { - - if constexpr (std::is_same() || std::is_same() || std::is_same() || - std::is_same()) { - FLS_ASSERT(!val_vec.empty(), "an empty vec", " "); - FLS_ASSERT(!rep_vec.empty(), "an empty vec", " "); - - n_t result {0}; - bool is_exc {false}; - - for (n_t i = 0; i < val_vec.size(); ++i) { - is_exc = is_exception(lower_bound, upper_bound, val_vec[i]); - if (!is_exc) { - continue; - } - /* It is an exception. - * Increase the number of exception by the repetition of this value. - */ - result += rep_vec[i]; - } - - return result; - } else { - FLS_ABORT("this is not supported!") - return 0; - } -} - -template -void typed_patch::b_patch_decompress(T* out_p, uint64_t* bitmap, T* exc_p) { - int c {0}; - unit_t unit; - for (size_t i {0}; i < CFG::BitMap::UNIT_C; ++i) { - unit = bitmap[i]; - while (unit != 0) { - uint64_t t = unit & -unit; - int r = __builtin_ctzll(unit); - out_p[i * 64 + r] = exc_p[c++]; - unit ^= t; - } - } -} - -template -void typed_patch::l_patch_decompress(T* in, rle_idx_t first, rle_idx_t c) { -} - -template -void typed_patch::l_patch_compress(T* in, T* exc, rle_idx_t arr[], rle_idx_t c, n_t max_range) { - // if (stt.exc_c == 0) { - // /* push the pos of first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); - // /* push the number of exceptions */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); - // - // stt.cur_des_arr += 2; - // - // return; - // } - // - // if (stt.exc_c == 1) { - // /* push the pos of first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_pos_arr, sizeof(pos_t)); - // /* push the number of exceptions */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); - // /* push the first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(exc_arr, sizeof(T)); - // = - // stt.cur_des_arr += 2; - // - // return; - // } - // - // /* push the pos of first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_pos_arr[0], sizeof(pos_t)); - // /* push the first exception */ - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(exc_arr + 0, sizeof(T)); - // - // T curr_pos = stt.exc_pos_arr[0]; - // T next_pos = stt.exc_pos_arr[1]; - // T max_range = 1 << stt.bw; - // pos_t new_c {0}; - // for (c_t i {1}; i < stt.exc_c; ++i) { - // curr_pos = stt.exc_pos_arr[i - 1]; - // next_pos = stt.exc_pos_arr[i]; - // while (patch::is_compulsory_exc(curr_pos, next_pos, max_range)) { - // /* Insert. */ - // in_arr[curr_pos] = max_range + (*base_p) - 1; - // curr_pos += max_range; - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(in_arr + curr_pos, sizeof(T)); - // new_c++; - // } - // in_arr[curr_pos] = (next_pos - curr_pos) + (*base_p) - 1; - // des_vec.arr_arr[stt.cur_des_arr + 0].UnsafeAppend(exc_arr + i, sizeof(T)); - // } - // in_arr[next_pos] = *(base_p); - // - // /* forth, push the number of exceptions */ - // stt.exc_c += new_c; - // des_vec.arr_arr[stt.cur_des_arr + 1].UnsafeAppend(&stt.exc_c, sizeof(pos_t)); -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_CTS(typed_patch) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/s_patch/CMakeLists.txt b/src/cor/prm/patch/s_patch/CMakeLists.txt deleted file mode 100644 index 461f49be..00000000 --- a/src/cor/prm/patch/s_patch/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_s_patch_prm - OBJECT - s_patch_decompress.cpp - s_patch_compress.cpp) - -target_compile_definitions(fls_s_patch_prm PRIVATE IS_SCALAR) -target_compile_options(fls_s_patch_prm PRIVATE "") -if (ENABLE_IWYU) - set_target_properties(fls_s_patch_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif () -set_target_properties(fls_s_patch_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/patch/s_patch/s_patch_compress.cpp b/src/cor/prm/patch/s_patch/s_patch_compress.cpp deleted file mode 100644 index 11d3a528..00000000 --- a/src/cor/prm/patch/s_patch/s_patch_compress.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/s_patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void s_patch_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - /* first, push the number of exceptions */ - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.exc_c, sizeof(uint16_t)); - - for (n_t i {0}; i < stt.exc_c; i++) { - /* [TODO], after recursive compression store bw bits of exception. */ - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&in_arr[stt.exc_pos_arr[i]], sizeof(T)); - des_vec.buf_arr[stt.cur_des_buff + 1].UnsafeAppend(&stt.exc_pos_arr[i], sizeof(uint16_t)); - } - - stt.cur_des_buff += 2; -} - -template -cmpr_fun_t s_patch::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return s_patch_compress; - } - if constexpr (std::is_same()) { - return s_patch_compress; - } - if constexpr (std::is_same()) { - return s_patch_compress; - } - if constexpr (std::is_same()) { - return s_patch_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, s_patch::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/patch/s_patch/s_patch_decompress.cpp b/src/cor/prm/patch/s_patch/s_patch_decompress.cpp deleted file mode 100644 index 0339ceb5..00000000 --- a/src/cor/prm/patch/s_patch/s_patch_decompress.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/patch/s_patch.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" - -namespace fastlanes { -template -static void s_patch_decompress(PageParam src, VecParam des, DecompressState& stt) { - FLS_ASSERT_NOT_NULL_POINTER(des.arr[stt.cur_des_arr]) - - auto* exc_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 1]); - auto* exc_pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_arr = reinterpret_cast(des.arr[stt.cur_des_arr]); - uint16_t exc_c = exc_pos_arr[0]; - - for (uint16_t i {0}; i < exc_c; ++i) { - auto next_pos = exc_pos_arr[i + 1]; - out_arr[next_pos] = exc_arr[i]; - } - - stt.cur_src_arr -= 2; - - FLS_PLOG_KEY_VALUE("exc_arr", ToStr::to_str(exc_arr, exc_c)); - FLS_PLOG_KEY_VALUE("after s_patch", ToStr::to_str(out_arr)); -} - -template -de_cmpr_fun_t s_patch::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return s_patch_decompress; - } - if constexpr (std::is_same()) { - return s_patch_decompress; - } - if constexpr (std::is_same()) { - return s_patch_decompress; - } - if constexpr (std::is_same()) { - return s_patch_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, s_patch::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rle/CMakeLists.txt b/src/cor/prm/rle/CMakeLists.txt deleted file mode 100644 index 81e2800f..00000000 --- a/src/cor/prm/rle/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_library(fls_rle_prm - OBJECT - rle_decompress.cpp - rle_compress.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_rle_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_rle_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/rle/rle_compress.cpp b/src/cor/prm/rle/rle_compress.cpp deleted file mode 100644 index 67d4c3dc..00000000 --- a/src/cor/prm/rle/rle_compress.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rle_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/transpose/transpose.hpp" - -namespace fastlanes { -template -static void rle_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - const auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 0].data()); - auto* pos_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 1].mutable_data()); - auto* tmp_pos_arr = reinterpret_cast(des_vec.buf_arr[stt.cur_des_buff + 1].mutable_data()); - - rle_idx_t pos_val; - for (n_t i {0}; i < vec_n_tup(); ++i) { - auto cur_val = in_arr[i]; - - if (i == 0) { - cur_val = in_arr[0]; - pos_val = 0; - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&cur_val, sizeof(T)); - tmp_pos_arr[i] = pos_val; - continue; - } - - if (cur_val != in_arr[i - 1]) { - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(&cur_val, sizeof(T)); - pos_val += 1; - } - tmp_pos_arr[i] = pos_val; - } - - generated::transpose::fallback::scalar::transpose_i(tmp_pos_arr, pos_arr); - - stt.cur_src_buff += 1; - stt.cur_des_buff += 1; - - FLS_PLOG_KEY_VALUE("pos_arr", ToStr::to_str(pos_arr)); - FLS_PLOG_KEY_VALUE("tmp_pos_arr", ToStr::to_str(tmp_pos_arr)); -} - -template -cmpr_fun_t rle::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return rle_compress; - } - if constexpr (std::is_same()) { - return rle_compress; - } - if constexpr (std::is_same()) { - return rle_compress; - } - if constexpr (std::is_same()) { - return rle_compress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, rle::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rle/rle_decompress.cpp b/src/cor/prm/rle/rle_decompress.cpp deleted file mode 100644 index 8db7da74..00000000 --- a/src/cor/prm/rle/rle_decompress.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "fls/cfg/cfg.hpp" -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rle_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include - -namespace fastlanes { -template -static void rle_decompress(PageParam src, VecParam des, DecompressState& stt) { - auto* in_pos_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - auto* out_pos_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); - auto* in_val_arr = reinterpret_cast(src.arr[stt.cur_des_arr - 1]); - auto* out_val_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 1]); - - // [FIXME] - // zero copy, - // not possible yet! after memory pool. - // out_val_arr = in_val_arr; - // out_pos_arr = in_pos_arr; - - std::memcpy(out_pos_arr, in_pos_arr, vec_bsz()); - /* Lest pos_t determine the size of val_arr. */ - std::memcpy(out_val_arr, in_val_arr, (in_pos_arr[1023] + 1) * sizeof(T)); - - FLS_PLOG_KEY_VALUE("in_val_arr", ToStr::to_str(in_val_arr)); - FLS_PLOG_KEY_VALUE("in_pos_arr", ToStr::to_str(in_pos_arr)); - - FLS_PLOG_KEY_VALUE("out_val_arr", ToStr::to_str(out_val_arr)); - FLS_PLOG_KEY_VALUE("out_pos_arr", ToStr::to_str(out_pos_arr)); - - stt.cur_src_arr -= 2; - stt.cur_des_arr -= 2; -} - -template -de_cmpr_fun_t rle::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return rle_decompress; - } - if constexpr (std::is_same()) { - return rle_decompress; - } - if constexpr (std::is_same()) { - return rle_decompress; - } - if constexpr (std::is_same()) { - return rle_decompress; - } - - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, rle::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rsum/CMakeLists.txt b/src/cor/prm/rsum/CMakeLists.txt deleted file mode 100644 index b506fec3..00000000 --- a/src/cor/prm/rsum/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_library(fls_rsum_prm - OBJECT - rsum_decompress.cpp - rsum_compress.cpp) - -if(ENABLE_IWYU) - set_target_properties(fls_rsum_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_rsum_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/rsum/rsum_compress.cpp b/src/cor/prm/rsum/rsum_compress.cpp deleted file mode 100644 index c6f1746a..00000000 --- a/src/cor/prm/rsum/rsum_compress.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rsum_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls/utl/util.hpp" -#include "fls_gen/unrsum/unrsum.hpp" - -namespace fastlanes { -template -static void rsum_compress(Vec& src_vec, Vec& des_vec, CompressState& stt) { - auto* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff + 0].mutable_data()); - - /* delta_bases_arr -> des[i + 0] */ - des_vec.buf_arr[stt.cur_des_buff + 0].UnsafeAppend(in_arr, 1024 / 8); - - /* rsum_data -> src[i + 0] */ - generated::unrsum::fallback::scalar::unrsum_inplace(in_arr); - - stt.cur_des_buff += 1; - - FLS_PLOG_KEY_VALUE("in_arr after unrsum", ToStr::to_str(in_arr)); -} - -template -cmpr_fun_t rsum::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } else if constexpr (std::is_same()) { - return rsum_compress; - } - - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, rsum::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/rsum/rsum_decompress.cpp b/src/cor/prm/rsum/rsum_decompress.cpp deleted file mode 100644 index 16af2f00..00000000 --- a/src/cor/prm/rsum/rsum_decompress.cpp +++ /dev/null @@ -1,51 +0,0 @@ -#include "fls/common/common.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/page/page.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/rsum_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls_gen/rsum/rsum.hpp" - -namespace fastlanes { -template -static void rsum_decompress(PageParam src, VecParam des, DecompressState& stt) { - auto* in_p = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); - auto* out_p = reinterpret_cast(des.arr[stt.cur_des_arr - 1]); - auto* delta_bases_arr = reinterpret_cast(src.arr[stt.cur_src_arr]); - - generated::rsum::fallback::scalar::rsum(in_p, out_p, delta_bases_arr); - - FLS_PLOG_KEY_VALUE("cur_vec", ToStr::to_str(reinterpret_cast(out_p))) - FLS_PLOG_KEY_VALUE("delta_bases_arr", ToStr::to_str(delta_bases_arr, 16)); - - stt.cur_src_arr -= 1; - stt.cur_des_arr -= 2; -} - -template -de_cmpr_fun_t rsum::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } else if constexpr (std::is_same()) { - return rsum_decompress; - } - - FLS_ABORT("THIS IT NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, rsum::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/trs/CMakeLists.txt b/src/cor/prm/trs/CMakeLists.txt deleted file mode 100644 index 10070655..00000000 --- a/src/cor/prm/trs/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_library(fls_trs_prm - OBJECT - trs_decompress.cpp - trs_compress.cpp) - -target_compile_definitions(fls_trs_prm PRIVATE IS_SCALAR) -target_compile_options(fls_trs_prm PRIVATE "") -if(ENABLE_IWYU) - set_target_properties(fls_trs_prm PROPERTIES CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) -endif() -set_target_properties(fls_trs_prm PROPERTIES CXX_CLANG_TIDY "") - -set(FASTLANES_OBJECT_FILES - ${FASTLANES_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/cor/prm/trs/trs_compress.cpp b/src/cor/prm/trs/trs_compress.cpp deleted file mode 100644 index decec203..00000000 --- a/src/cor/prm/trs/trs_compress.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/trs_prm.hpp" -#include "fls/logger/logger.hpp" -#include "fls/utl/to_str.hpp" -#include "fls_gen/transpose/transpose.hpp" -#include - -namespace fastlanes { -template -static void trs_cmpr(Vec& src_vec, Vec& des_vec, CompressState& stt) { - T* in_arr = reinterpret_cast(src_vec.buf_arr[stt.cur_src_buff].mutable_data()); - - FLS_PLOG_KEY_VALUE("before transpose", ToStr::to_str(in_arr)); - /* Fixme: not efficient. implement inplace_transpose */ - T tmp[vec_n_tup()]; - generated::transpose::fallback::scalar::transpose_i(in_arr, tmp); - std::memcpy(in_arr, tmp, vec_bsz()); - FLS_PLOG_KEY_VALUE("after transpose", ToStr::to_str(in_arr)); -} - -template -cmpr_fun_t trs::ResolveCompressFunc() { - if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same()) { - return trs_cmpr; - } else if constexpr (std::is_same_v) { - return trs_cmpr; - } - - FLS_ABORT("THIS IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(cmpr_fun_t, trs::ResolveCompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/trs/trs_decompress.cpp b/src/cor/prm/trs/trs_decompress.cpp deleted file mode 100644 index 23a549ff..00000000 --- a/src/cor/prm/trs/trs_decompress.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "fls/cor/eng/compressor.hpp" -#include "fls/cor/eng/decompressor.hpp" -#include "fls/cor/lyt/vec.hpp" -#include "fls/cor/prm/trs_prm.hpp" - -namespace fastlanes { -template -static void trs_decmpr(PageParam src, VecParam des, DecompressState& stt) { - // auto *in_arr = reinterpret_cast(src.arr[stt.cur_src_arr - 0]); - // auto *out_arr = reinterpret_cast(des.arr[stt.cur_des_arr - 0]); -} - -template -de_cmpr_fun_t trs::ResolveDecompressFunc() { - if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } else if constexpr (std::is_same()) { - return trs_decmpr; - } - - FLS_ABORT("THIS IS NOT SUPPORTED") - return nullptr; -} - -/*---------------------------------------------------------------------------------------------------------------------\ - * Specialization -\---------------------------------------------------------------------------------------------------------------------*/ -FLS_ALL_FTS(de_cmpr_fun_t, trs::ResolveDecompressFunc) - -} // namespace fastlanes diff --git a/src/cor/prm/type.cpp b/src/cor/prm/type.cpp deleted file mode 100644 index 344cdac4..00000000 --- a/src/cor/prm/type.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "fls/cor/prm/prm_type.hpp" -#include - -namespace fastlanes { -std::string to_string(prm_t prm_t) { - switch (prm_t) { - case prm_t::INVALID: - break; - case prm_t::ANALYZE: - return {"ANALYZE"}; - case prm_t::PA_ANALYZE: - return {"PA_ANALYZE"}; - case prm_t::MEM_CPY: - return {"MEM_CPY"}; - case prm_t::TRS: - return {"TRS"}; - case prm_t::MEM_TRS: - return {"MEM_TRS"}; - case prm_t::LL_PATCH: - return {"LL_PATCH"}; - case prm_t::B_FFOR: - return {"B_FFOR"}; - case prm_t::RLE: - case prm_t::RSUM: - case prm_t::BITPACK: - case prm_t::FACTOR: - case prm_t::FSST: - case prm_t::B_PATCH: - case prm_t::S_PATCH: - case prm_t::FFOR: - case prm_t::ZER_CPY: - // TODO [FIXME] - default: - break; - } - - return {}; -} -} // namespace fastlanes diff --git a/src/expression/CMakeLists.txt b/src/expression/CMakeLists.txt index b8556931..be234fa3 100644 --- a/src/expression/CMakeLists.txt +++ b/src/expression/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(fls_expression alp_expression.cpp analyze_operator.cpp # CMakeLists.txt + data_parallelize_patch_operator.cpp cross_rle_operator.cpp data_type.cpp decoding_operator.cpp diff --git a/src/expression/alp_expression.cpp b/src/expression/alp_expression.cpp index 5a40d123..40cc2fe2 100644 --- a/src/expression/alp_expression.cpp +++ b/src/expression/alp_expression.cpp @@ -4,6 +4,7 @@ #include "fls/expression/decoding_operator.hpp" #include "fls/expression/physical_expression.hpp" #include "fls/primitive/bitpack/bitpack.hpp" +#include "fls/primitive/patch/patch.hpp" #include "fls/reader/column_view.hpp" #include "fls/reader/segment.hpp" #include "fls/unffor.hpp" @@ -30,6 +31,9 @@ enc_alp_opr::enc_alp_opr(const PhysicalExpr& expr, exp_segment = make_unique(); pos_segment = make_unique(); n_exp_segment = make_unique(); + // + out_count_segment = make_unique(); + out_offset_segment = make_unique(); } template @@ -59,6 +63,11 @@ void enc_alp_opr::Encode() { alp::encoder::analyze_ffor(encoded_arr, alp_state.bit_width, &base); ffor::ffor(encoded_arr, ffor_arr, alp_state.bit_width, &base); + if (data_parallelize) { + Patch::data_parallelize( + exc_arr, pos_arr, alp_state.n_exceptions, out_exc_arr, out_offset, out_pos_arr, out_count); + } + const n_t bytes = calculate_bitpacked_vector_size(alp_state.bit_width); const n_t exc_arr_size = calculate_alp_exception_vector_size(alp_state.n_exceptions); const n_t pos_arr_size = calculate_alp_pos_vector_size(alp_state.n_exceptions); @@ -66,11 +75,28 @@ void enc_alp_opr::Encode() { bw_segment->Flush(&alp_state.bit_width, sizeof(bw_t)); base_segment->Flush(&base, sizeof(typename alp::inner_t::st)); ffor_segment->Flush(ffor_arr, bytes); - exception_segment->Flush(exc_arr, exc_arr_size); + + if (data_parallelize) { + exception_segment->Flush(out_exc_arr, exc_arr_size); + } else { + exception_segment->Flush(exc_arr, exc_arr_size); + } + fac_segment->Flush(&alp_state.fac, sizeof(uint8_t)); exp_segment->Flush(&alp_state.exp, sizeof(uint8_t)); - pos_segment->Flush(pos_arr, pos_arr_size); + + if (data_parallelize) { + pos_segment->Flush(out_pos_arr, pos_arr_size); + } else { + pos_segment->Flush(pos_arr, pos_arr_size); + } + n_exp_segment->Flush(&alp_state.n_exceptions, sizeof(uint16_t)); + + if (data_parallelize) { + out_count_segment->Flush(out_count, 32); + out_offset_segment->Flush(out_offset, 64); + }; } template @@ -84,6 +110,10 @@ void enc_alp_opr::MoveSegments(vector>& segments) { segments.push_back(std::move(exp_segment)); segments.push_back(std::move(pos_segment)); segments.push_back(std::move(n_exp_segment)); + if (data_parallelize) { + segments.push_back(std::move(out_count_segment)); + segments.push_back(std::move(out_offset_segment)); + } } template struct enc_alp_opr; diff --git a/src/expression/data_parallelize_patch_operator.cpp b/src/expression/data_parallelize_patch_operator.cpp new file mode 100644 index 00000000..0c139a63 --- /dev/null +++ b/src/expression/data_parallelize_patch_operator.cpp @@ -0,0 +1,34 @@ +#include "fls/expression/data_parallelize_patch_operator.hpp" +#include "fls/common/string.hpp" +#include "fls/cor/lyt/buf.hpp" +#include "fls/expression/alp_expression.hpp" +#include "fls/expression/decoding_operator.hpp" +#include "fls/expression/interpreter.hpp" +#include "fls/expression/physical_expression.hpp" +#include "fls/primitive/patch/patch.hpp" +#include "fls/reader/column_view.hpp" +#include "fls/reader/segment.hpp" +#include "fls/unffor.hpp" +#include + +namespace fastlanes { +/*--------------------------------------------------------------------------------------------------------------------*\ + * enc_data_parallel_patch_opr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +enc_data_parallel_patch_opr::enc_data_parallel_patch_opr(const PhysicalExpr& expr, + const col_pt& column, + ColumnDescriptorT& column_descriptor, + InterpreterState& state) { + visit(overloaded { + [&](const sp>& opr) { opr->data_parallelize = true; }, + [&](std::monostate&) { FLS_UNREACHABLE(); }, + [&](auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg); }, + }, + expr.operators[state.cur_operator++]); +} + +template struct enc_data_parallel_patch_opr; +template struct enc_data_parallel_patch_opr; + +} // namespace fastlanes diff --git a/src/expression/expression_executor.cpp b/src/expression/expression_executor.cpp index 3b1c771e..13c14dfe 100644 --- a/src/expression/expression_executor.cpp +++ b/src/expression/expression_executor.cpp @@ -1,6 +1,7 @@ #include "fls/expression/expression_executor.hpp" #include "fls/expression/alp_expression.hpp" #include "fls/expression/analyze_operator.hpp" +#include "fls/expression/data_parallelize_patch_operator.hpp" #include "fls/expression/decoding_operator.hpp" #include "fls/expression/encoding_operator.hpp" #include "fls/expression/frequency_operator.hpp" @@ -219,6 +220,9 @@ struct operator_visitor { opr->to_validitymask(); } // + template + void operator()(sp>& opr) { + } void operator()(std::monostate&) { FLS_UNREACHABLE(); } diff --git a/src/expression/interpreter.cpp b/src/expression/interpreter.cpp index a587f9f8..aadb422d 100644 --- a/src/expression/interpreter.cpp +++ b/src/expression/interpreter.cpp @@ -5,6 +5,7 @@ #include "fls/expression/alp_expression.hpp" #include "fls/expression/analyze_operator.hpp" #include "fls/expression/cross_rle_operator.hpp" +#include "fls/expression/data_parallelize_patch_operator.hpp" #include "fls/expression/decoding_operator.hpp" #include "fls/expression/dict_expression.hpp" #include "fls/expression/encoding_operator.hpp" @@ -348,6 +349,22 @@ void make_enc_alp_expr(PhysicalExpr& physical_expr, operators.emplace_back(make_shared>(physical_expr, column, column_descriptor, state)); } +/*--------------------------------------------------------------------------------------------------------------------*\ + * make_galp_expr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +void make_enc_galp_expr(PhysicalExpr& physical_expr, + const rowgroup_pt& rowgroup, + ColumnDescriptorT& column_descriptor, + InterpreterState& state) { + auto& operators = physical_expr.operators; + const auto& column = rowgroup[column_descriptor.idx]; + + operators.emplace_back(make_shared>(physical_expr, column, column_descriptor, state)); + operators.emplace_back( + make_shared>(physical_expr, column, column_descriptor, state)); +} + /*--------------------------------------------------------------------------------------------------------------------*\ * make_rle_expr \*--------------------------------------------------------------------------------------------------------------------*/ @@ -613,6 +630,14 @@ sp Interpreter::Encoding::Interpret(ColumnDescriptorT& column_desc make_enc_alp_expr(*physical_expr, physical_rowgroup, column_descriptor, state); break; } + case EXP_GALP_FLT: { + make_enc_galp_expr(*physical_expr, physical_rowgroup, column_descriptor, state); + break; + } + case EXP_GALP_DBL: { + make_enc_galp_expr(*physical_expr, physical_rowgroup, column_descriptor, state); + break; + } case EXP_ALP_RD_DBL: { make_enc_alp_rd_expr(*physical_expr, physical_rowgroup, column_descriptor, state); break; @@ -1125,6 +1150,15 @@ void make_dec_alp_expr(PhysicalExpr& physical_expr, const ColumnView& column_vie physical_expr.operators.emplace_back(make_shared>(column_view, state)); } +/*--------------------------------------------------------------------------------------------------------------------*\ + * make_dec_galp_expr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +void make_dec_galp_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { + state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + physical_expr.operators.emplace_back(make_shared>(column_view, state)); +} + /*--------------------------------------------------------------------------------------------------------------------*\ * make_dec_alp_rd_expr \*--------------------------------------------------------------------------------------------------------------------*/ @@ -1616,6 +1650,14 @@ void Interpreter::Decoding::Interpret(const ColumnDescriptorT& column_descriptor make_dec_alp_expr(physical_expr, column_view, state); break; } + case EXP_GALP_FLT: { + make_dec_galp_expr(physical_expr, column_view, state); + break; + } + case EXP_GALP_DBL: { + make_dec_galp_expr(physical_expr, column_view, state); + break; + } case EXP_ALP_RD_DBL: { make_dec_alp_rd_expr(physical_expr, column_view, state); break; diff --git a/src/expression/physical_expression.cpp b/src/expression/physical_expression.cpp index b4601b1e..e4f3ab5b 100644 --- a/src/expression/physical_expression.cpp +++ b/src/expression/physical_expression.cpp @@ -3,6 +3,7 @@ #include "fls/expression/alp_expression.hpp" #include "fls/expression/analyze_operator.hpp" #include "fls/expression/cross_rle_operator.hpp" +#include "fls/expression/data_parallelize_patch_operator.hpp" #include "fls/expression/decoding_operator.hpp" #include "fls/expression/dict_expression.hpp" #include "fls/expression/encoding_operator.hpp" @@ -218,6 +219,10 @@ struct point_to_visitor { template void operator()(const sp>& opr) { } + // + template + void operator()(const sp>& opr) { + } void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } @@ -360,7 +365,10 @@ struct flush_segments_visitor { void operator()(const sp& opr) { opr->MoveSegments(segments); } - + // + template + void operator()(const sp>& opr) { + } void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } @@ -514,6 +522,10 @@ struct extract_segments_visitor { opr->MoveSegments(segments); } // + template + void operator()(const sp>& opr) { + } + // void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } @@ -651,6 +663,10 @@ struct finalize_operators_visitor { void operator()(const sp& opr) { } // + template + void operator()(const sp>& opr) { + } + // void operator()(const auto& arg) { FLS_UNREACHABLE_WITH_TYPE(arg) } diff --git a/src/expression/rpn.cpp b/src/expression/rpn.cpp index e53fe994..8deca12a 100644 --- a/src/expression/rpn.cpp +++ b/src/expression/rpn.cpp @@ -318,6 +318,12 @@ string token_to_string(OperatorToken token) { return "EXP_RLE_U08_SLPATCH_U16"; // 152 case OperatorToken::EXP_DELTA_U08: return "EXP_DELTA_U08"; // 153 + case OperatorToken::EXP_RLE_U08_U16: + return "EXP_RLE_U08_U16"; // 154 + case OperatorToken::EXP_GALP_FLT: + return "EXP_GALP_FLT"; // 155 + case OperatorToken::EXP_GALP_DBL: + return "EXP_GALP_DBL"; // 156 default: FLS_UNREACHABLE() } diff --git a/src/include/fls/cuda/common.hpp b/src/include/fls/cuda/common.hpp new file mode 100644 index 00000000..c8ba4d33 --- /dev/null +++ b/src/include/fls/cuda/common.hpp @@ -0,0 +1,32 @@ +#ifndef FLS_CUDA_COMMON_HPP +#define FLS_CUDA_COMMON_HPP + +#include "fls/cfg/cfg.hpp" +#include "fls/common/alias.hpp" +#include "fls/cuda/config.hpp" + +namespace fastlanes::cuda { + +template +constexpr n_t bits_in_object() { + return sizeof(PT) * CHAR_BIT; +} + +template +constexpr n_t get_lane_bitwidth() { + return bits_in_object(); +} + +template +constexpr n_t get_n_lanes() { + return Config::REGISTER_WIDTH / get_lane_bitwidth(); +} + +template +constexpr n_t get_values_per_lane() { + return CFG::VEC_SZ / get_n_lanes(); +} + +} // namespace fastlanes::cuda + +#endif // FLS_CUDA_COMMON_HPP \ No newline at end of file diff --git a/src/include/fls/cuda/config.hpp b/src/include/fls/cuda/config.hpp new file mode 100644 index 00000000..9342caa7 --- /dev/null +++ b/src/include/fls/cuda/config.hpp @@ -0,0 +1,16 @@ +#ifndef FLS_CUDA_CONFIG_HPP +#define FLS_CUDA_CONFIG_HPP + +#include "fls/common/alias.hpp" // for hdr_field_t + +namespace fastlanes::cuda { +class Config { +public: + explicit Config(); // +public: + static constexpr n_t REGISTER_WIDTH = 1024; // +}; + +} // namespace fastlanes::cuda + +#endif // FLS_CUDA_CONFIG_HPP diff --git a/src/include/fls/expression/alp_expression.hpp b/src/include/fls/expression/alp_expression.hpp index 4215fae9..8c9a3326 100644 --- a/src/include/fls/expression/alp_expression.hpp +++ b/src/include/fls/expression/alp_expression.hpp @@ -3,6 +3,7 @@ #include "alp.hpp" #include "fls/cfg/cfg.hpp" +#include "fls/cuda/common.hpp" #include "fls/reader/segment.hpp" #include "fls/table/chunk.hpp" #include "fls/table/rowgroup.hpp" @@ -42,6 +43,10 @@ struct enc_alp_opr { alignas(64) uint16_t pos_arr[CFG::VEC_SZ]; alignas(64) PT exc_arr[CFG::VEC_SZ]; + // + bool data_parallelize {false}; + + // cuda up ffor_segment; up base_segment; up bw_segment; @@ -50,6 +55,16 @@ struct enc_alp_opr { up exp_segment; up pos_segment; up n_exp_segment; + + // + // arrays + alignas(64) uint8_t out_count[cuda::get_n_lanes()]; + alignas(64) uint16_t out_offset[cuda::get_n_lanes()]; + alignas(64) uint16_t out_pos_arr[CFG::VEC_SZ]; + alignas(64) PT out_exc_arr[CFG::VEC_SZ]; + + up out_count_segment; + up out_offset_segment; }; /*--------------------------------------------------------------------------------------------------------------------*\ diff --git a/src/include/fls/expression/data_parallelize_patch_operator.hpp b/src/include/fls/expression/data_parallelize_patch_operator.hpp new file mode 100644 index 00000000..18de8340 --- /dev/null +++ b/src/include/fls/expression/data_parallelize_patch_operator.hpp @@ -0,0 +1,32 @@ +#ifndef FLS_EXPRESSION_DATA_PARALLEL_PATCH_OPERATOR_HPP +#define FLS_EXPRESSION_DATA_PARALLEL_PATCH_OPERATOR_HPP + +#include "fls/cuda/common.hpp" +#include "fls/reader/segment.hpp" +#include "fls/table/chunk.hpp" +#include "fls/table/rowgroup.hpp" + +namespace fastlanes { +/*--------------------------------------------------------------------------------------------------------------------*/ +class Segment; +struct ColumnDescriptorT; +class PhysicalExpr; +struct InterpreterState; +class ColumnView; +struct InterpreterState; +/*--------------------------------------------------------------------------------------------------------------------*/ + +/*--------------------------------------------------------------------------------------------------------------------*\ + * enc_alp_opr +\*--------------------------------------------------------------------------------------------------------------------*/ +template +struct enc_data_parallel_patch_opr { + explicit enc_data_parallel_patch_opr(const PhysicalExpr& expr, + const col_pt& column, + ColumnDescriptorT& column_descriptor, + InterpreterState& state); +}; + +} // namespace fastlanes + +#endif // FLS_EXPRESSION_DATA_PARALLEL_PATCH_OPERATOR_HPP diff --git a/src/include/fls/expression/physical_expression.hpp b/src/include/fls/expression/physical_expression.hpp index c2275f46..afbc805a 100644 --- a/src/include/fls/expression/physical_expression.hpp +++ b/src/include/fls/expression/physical_expression.hpp @@ -89,6 +89,8 @@ template struct dec_cross_rle_opr; struct enc_validitymask_opr; struct dec_validitymask_opr; +template +struct enc_data_parallel_patch_opr; /*--------------------------------------------------------------------------------------------------------------------*/ using physical_operator = variant, sp>, sp>, - sp> + sp>, + // + // DATA Parallelize Patch + sp>, + sp> // >; diff --git a/src/include/fls/footer/operator_token_generated.h b/src/include/fls/footer/operator_token_generated.h index edb96fe1..7ac5c01b 100644 --- a/src/include/fls/footer/operator_token_generated.h +++ b/src/include/fls/footer/operator_token_generated.h @@ -167,10 +167,12 @@ enum class OperatorToken : uint16_t { EXP_FFOR_SLPATCH_U08 = 151, EXP_RLE_U08_SLPATCH_U16 = 152, EXP_DELTA_U08 = 153, - EXP_RLE_U08_U16 = 154 + EXP_RLE_U08_U16 = 154, + EXP_GALP_FLT = 155, + EXP_GALP_DBL = 156 }; -inline const OperatorToken (&EnumValuesOperatorToken())[155] { +inline const OperatorToken (&EnumValuesOperatorToken())[157] { static const OperatorToken values[] = {OperatorToken::INVALID, OperatorToken::EXP_EQUAL, OperatorToken::EXP_CONSTANT_I64, @@ -325,12 +327,14 @@ inline const OperatorToken (&EnumValuesOperatorToken())[155] { OperatorToken::EXP_FFOR_SLPATCH_U08, OperatorToken::EXP_RLE_U08_SLPATCH_U16, OperatorToken::EXP_DELTA_U08, - OperatorToken::EXP_RLE_U08_U16}; + OperatorToken::EXP_RLE_U08_U16, + OperatorToken::EXP_GALP_FLT, + OperatorToken::EXP_GALP_DBL}; return values; } inline const char* const* EnumNamesOperatorToken() { - static const char* const names[156] = {"INVALID", + static const char* const names[158] = {"INVALID", "EXP_EQUAL", "EXP_CONSTANT_I64", "EXP_CONSTANT_I32", @@ -485,12 +489,14 @@ inline const char* const* EnumNamesOperatorToken() { "EXP_RLE_U08_SLPATCH_U16", "EXP_DELTA_U08", "EXP_RLE_U08_U16", + "EXP_GALP_FLT", + "EXP_GALP_DBL", nullptr}; return names; } inline const char* EnumNameOperatorToken(OperatorToken e) { - if (::flatbuffers::IsOutRange(e, OperatorToken::INVALID, OperatorToken::EXP_RLE_U08_U16)) + if (::flatbuffers::IsOutRange(e, OperatorToken::INVALID, OperatorToken::EXP_GALP_DBL)) return ""; const size_t index = static_cast(e); return EnumNamesOperatorToken()[index]; diff --git a/src/include/fls/primitive/patch/patch.hpp b/src/include/fls/primitive/patch/patch.hpp new file mode 100644 index 00000000..e8080db0 --- /dev/null +++ b/src/include/fls/primitive/patch/patch.hpp @@ -0,0 +1,22 @@ +#ifndef FLS_PRIMITIVE_PATCH_PATCH_HPP +#define FLS_PRIMITIVE_PATCH_PATCH_HPP + +#include "fls/common/common.hpp" + +namespace fastlanes { +n_t calculate_bitpacked_vector_size(bw_t bw); + +template +class Patch { +public: + static void data_parallelize(const PT* in_exc_arr, + const uint16_t* in_pos_arr, + n_t n_exceptions, + PT* out_exc_arr, + uint16_t* out_offset, + uint16_t* out_pos_arr, + uint8_t* out_count); +}; + +} // namespace fastlanes +#endif // FLS_PRIMITIVE_PATCH_PATCH_HPP diff --git a/src/primitive/CMakeLists.txt b/src/primitive/CMakeLists.txt index ba93b886..8ebd50cc 100644 --- a/src/primitive/CMakeLists.txt +++ b/src/primitive/CMakeLists.txt @@ -3,6 +3,7 @@ add_subdirectory(copy) add_subdirectory(fls_memset) add_subdirectory(fsst) add_subdirectory(fsst12) +add_subdirectory(patch) add_subdirectory(predicate) add_subdirectory(rle) add_subdirectory(untraspose) diff --git a/src/primitive/patch/CMakeLists.txt b/src/primitive/patch/CMakeLists.txt new file mode 100644 index 00000000..8c549b8e --- /dev/null +++ b/src/primitive/patch/CMakeLists.txt @@ -0,0 +1,20 @@ +add_library(fls_primitive_patch + OBJECT + patch.cpp +) + + +set(FASTLANES_OBJECT_FILES + ${FASTLANES_OBJECT_FILES} $ + PARENT_SCOPE) + +if (ENABLE_IWYU) + set_property(TARGET fls_primitive_patch PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_path}) +endif () + +target_compile_options(fls_primitive_patch PRIVATE -Wno-macro-redefined) + +target_link_libraries(fls_primitive_patch + PUBLIC + FastLanes::headers +) diff --git a/src/primitive/patch/patch.cpp b/src/primitive/patch/patch.cpp new file mode 100644 index 00000000..5d2b0f21 --- /dev/null +++ b/src/primitive/patch/patch.cpp @@ -0,0 +1,46 @@ +#include "fls/primitive/patch/patch.hpp" +#include "fls/cuda/common.hpp" + +namespace fastlanes { + +template +void Patch::data_parallelize(const PT* __restrict in_exc_arr, + const uint16_t* __restrict in_pos_arr, + n_t n_exceptions, + PT* __restrict out_exc_arr, + uint16_t* __restrict out_offset, + uint16_t* __restrict out_pos_arr, + uint8_t* __restrict out_count) { + // + constexpr n_t N_LANES = cuda::get_n_lanes(); + static_assert((N_LANES & (N_LANES - 1)) == 0, "N_LANES must be a power of two"); + constexpr n_t MASK = N_LANES - 1; + + // + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_count[lane] = 0; + } + + for (n_t i = 0; i < n_exceptions; ++i) { + ++out_count[in_pos_arr[i] & MASK]; + } + + // + uint16_t running = 0; + for (n_t lane = 0; lane < N_LANES; ++lane) { + out_offset[lane] = running; + running += out_count[lane]; + } + + for (n_t i = 0; i < n_exceptions; ++i) { + const n_t lane = in_pos_arr[i] & MASK; + const uint16_t dst = out_offset[lane]++; // advance only the cursor + out_exc_arr[dst] = in_exc_arr[i]; + out_pos_arr[dst] = in_pos_arr[i]; + } +} + +template class Patch; +template class Patch; + +} // namespace fastlanes \ No newline at end of file diff --git a/test/src/expression_tests/CMakeLists.txt b/test/src/expression_tests/CMakeLists.txt index ca9ec939..fb978304 100644 --- a/test/src/expression_tests/CMakeLists.txt +++ b/test/src/expression_tests/CMakeLists.txt @@ -1,30 +1,15 @@ -add_executable(test_frequency frequency_test.cpp) -target_link_libraries(test_frequency PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_frequency COMMAND $) - -add_executable(test_rle rle_test.cpp) -target_link_libraries(test_rle PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_rle COMMAND $) - -add_executable(test_cross_rle cross_rle_test.cpp) -target_link_libraries(test_cross_rle PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_cross_rle COMMAND $) - -add_executable(test_alp alp_test.cpp) -target_link_libraries(test_alp PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_alp COMMAND $) - -add_executable(test_fsst_expr fsst_test.cpp) -target_link_libraries(test_fsst_expr PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_fsst_expr COMMAND $) - -add_executable(test_uncompressed uncompressed_test.cpp) -target_link_libraries(test_uncompressed PUBLIC gtest_main gmock_main FastLanes) -add_test(NAME test_uncompressed COMMAND $) - -fls_enable_sanitizers(test_frequency) -fls_enable_sanitizers(test_rle) -fls_enable_sanitizers(test_cross_rle) -fls_enable_sanitizers(test_alp) -fls_enable_sanitizers(test_fsst_expr) -fls_enable_sanitizers(test_uncompressed) +macro(fls_add_test NAME SRC) + add_executable(test_${NAME} ${SRC}) + target_link_libraries(test_${NAME} PUBLIC gtest_main gmock_main FastLanes) + add_test(NAME test_${NAME} COMMAND $) + fls_enable_sanitizers(test_${NAME}) +endmacro() + +fls_add_test(alp alp_test.cpp) +# +fls_add_test(cross_rle cross_rle_test.cpp) +fls_add_test(frequency frequency_test.cpp) +fls_add_test(fsst_expr fsst_test.cpp) +fls_add_test(galp galp_test.cpp) +fls_add_test(rle rle_test.cpp) +fls_add_test(uncompressed uncompressed_test.cpp) diff --git a/test/src/expression_tests/galp_test.cpp b/test/src/expression_tests/galp_test.cpp new file mode 100644 index 00000000..f113712c --- /dev/null +++ b/test/src/expression_tests/galp_test.cpp @@ -0,0 +1,57 @@ +#include "fls_tester.hpp" + +namespace fastlanes { + +// ——— FLOAT tests ——————————————————————————————————————————————— + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_ONE_VECTOR) { + TestCorrectness(galp::float_galp_one_vector, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_SINGLE_COLUMN) { + TestCorrectness(galp::float_galp_single_column, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_CONSTANT_ONE_VECTOR) { + TestCorrectness(galp::float_constant_one_vector, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_CONSTANT_SINGLE_COLUMN) { + TestCorrectness(galp::float_constant_single_column, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_POSITIVE_INF_ONE_VECTOR) { + TestCorrectness(galp::float_positive_inf_one_vector, {OperatorToken::EXP_GALP_FLT}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_FLOAT_POSITIVE_INF_SINGLE_COLUMN) { + TestCorrectness(galp::float_positive_inf_single_column, {OperatorToken::EXP_GALP_FLT}); +} + +// ——— DOUBLE tests ——————————————————————————————————————————————— + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_ONE_VECTOR) { + TestCorrectness(galp::double_galp_one_vector, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_SINGLE_COLUMN) { + TestCorrectness(galp::double_galp_single_column, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_CONSTANT_ONE_VECTOR) { + TestCorrectness(galp::double_constant_one_vector, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_CONSTANT_SINGLE_COLUMN) { + TestCorrectness(galp::double_constant_single_column, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_POSITIVE_INF_ONE_VECTOR) { + TestCorrectness(galp::double_positive_inf_one_vector, {OperatorToken::EXP_GALP_DBL}); +} + +TEST_F(FastLanesReaderTester, TEST_GALP_DOUBLE_POSITIVE_INF_SINGLE_COLUMN) { + TestCorrectness(galp::double_positive_inf_single_column, {OperatorToken::EXP_GALP_DBL}); +} + +} // namespace fastlanes diff --git a/test/src/primitive_tests/CMakeLists.txt b/test/src/primitive_tests/CMakeLists.txt index 6c12846d..336535d2 100644 --- a/test/src/primitive_tests/CMakeLists.txt +++ b/test/src/primitive_tests/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable(primitive_test fsst_test.cpp + patch_test.cpp ) target_link_libraries(primitive_test diff --git a/test/src/primitive_tests/patch_test.cpp b/test/src/primitive_tests/patch_test.cpp new file mode 100644 index 00000000..8bdc965b --- /dev/null +++ b/test/src/primitive_tests/patch_test.cpp @@ -0,0 +1,163 @@ +// test/src/primitive_tests/patch_test.cpp +// Google Test suite for fastlanes::Patch::data_parallelize +// Updated for API that returns **per‑lane counts (uint8_t) and offsets (uint16_t)**. +// Scenarios: zero exceptions, single exception, one‑per‑lane, 1024 sequential. + +#include "fls/cuda/common.hpp" +#include "fls/primitive/patch/patch.hpp" +#include +#include + +namespace fastlanes { + +// ----------------------------------------------------------------------------- +// Typed‑test fixture +// ----------------------------------------------------------------------------- + +template +class PatchDataParallelizeTest : public ::testing::Test { +protected: + Patch patch; +}; + +using TestTypes = ::testing::Types; +TYPED_TEST_SUITE(PatchDataParallelizeTest, TestTypes); + +// ----------------------------------------------------------------------------- +// 1. Zero exceptions ---------------------------------------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, HandlesZeroExceptions) { + using PT = TypeParam; + + constexpr n_t n_exceptions = 0; + std::vector dummy_exc(1); + std::vector dummy_pos(1); + + std::vector out_exc(1); + std::vector out_pos(1); + const n_t N_LANES = cuda::get_n_lanes(); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize(dummy_exc.data(), + dummy_pos.data(), + n_exceptions, + out_exc.data(), + lane_end.data(), + out_pos.data(), + lane_cnt.data()); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + ASSERT_EQ(lane_cnt[lane], 0); + ASSERT_EQ(lane_end[lane], static_cast(0)); // inclusive prefix + } +} + +// ----------------------------------------------------------------------------- +// 2. Single exception routed correctly ---------------------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, SingleExceptionRoutedToCorrectLane) { + using PT = TypeParam; + + const std::vector in_exc = {42}; + const std::vector in_pos = {3}; + constexpr n_t n_exceptions = 1; + + std::vector out_exc(n_exceptions); + std::vector out_pos(n_exceptions); + const n_t N_LANES = cuda::get_n_lanes(); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize( + in_exc.data(), in_pos.data(), n_exceptions, out_exc.data(), lane_end.data(), out_pos.data(), lane_cnt.data()); + + ASSERT_EQ(out_exc[0], in_exc[0]); + ASSERT_EQ(out_pos[0], in_pos[0]); + + const n_t exc_lane = in_pos[0] & (N_LANES - 1); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + // counts + ASSERT_EQ(lane_cnt[lane], static_cast(lane == exc_lane)); + + // inclusive offset: 0 for lanes < exc_lane, 1 otherwise + const uint16_t expected_end = (lane < exc_lane) ? 0 : 1; + ASSERT_EQ(lane_end[lane], expected_end); + } +} + +// ----------------------------------------------------------------------------- +// 3. One exception per lane --------------------------------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, DistributesAcrossAllLanes) { + using PT = TypeParam; + const n_t N_LANES = cuda::get_n_lanes(); + + std::vector in_exc; + std::vector in_pos; + in_exc.reserve(N_LANES); + in_pos.reserve(N_LANES); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + in_exc.push_back(static_cast(lane + 10)); + in_pos.push_back(static_cast(lane)); + } + + const n_t n_exceptions = N_LANES; + std::vector out_exc(n_exceptions); + std::vector out_pos(n_exceptions); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize( + in_exc.data(), in_pos.data(), n_exceptions, out_exc.data(), lane_end.data(), out_pos.data(), lane_cnt.data()); + + for (n_t lane = 0; lane < N_LANES; ++lane) { + ASSERT_EQ(lane_cnt[lane], 1); + ASSERT_EQ(lane_end[lane], static_cast(lane + 1)); // inclusive + } + + for (n_t i = 0; i < n_exceptions; ++i) { + ASSERT_EQ(out_exc[i], in_exc[i]); + ASSERT_EQ(out_pos[i], in_pos[i]); + } +} + +// ----------------------------------------------------------------------------- +// 4. 1024 sequential exceptions (offset‑wrap stress) -------------------------- +// ----------------------------------------------------------------------------- + +TYPED_TEST(PatchDataParallelizeTest, HandlesThousandTwentyFourSequential) { + using PT = TypeParam; + constexpr n_t n_exceptions = 1024; + + std::vector in_exc(n_exceptions); + std::vector in_pos(n_exceptions); + for (n_t i = 0; i < n_exceptions; ++i) { + in_exc[i] = static_cast(i); + in_pos[i] = static_cast(i); + } + + std::vector out_exc(n_exceptions); + std::vector out_pos(n_exceptions); + const n_t N_LANES = cuda::get_n_lanes(); + std::vector lane_end(N_LANES, 0); + std::vector lane_cnt(N_LANES, 0); + + Patch().data_parallelize( + in_exc.data(), in_pos.data(), n_exceptions, out_exc.data(), lane_end.data(), out_pos.data(), lane_cnt.data()); + + n_t begin = 0; + for (n_t lane = 0; lane < N_LANES; ++lane) { + const n_t expected_end = begin + lane_cnt[lane]; + ASSERT_EQ(lane_end[lane], static_cast(expected_end)); + begin = expected_end; + } + ASSERT_EQ(begin, n_exceptions); // final end == total exceptions +} + +} // namespace fastlanes From 8a981dad2a60783b87bff8cc2f8cb2ed0c5af752 Mon Sep 17 00:00:00 2001 From: azimafroozeh Date: Tue, 22 Jul 2025 23:15:31 +0200 Subject: [PATCH 2/3] remove cuda workflow for now --- .github/workflows/cuda.yml | 58 -------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 .github/workflows/cuda.yml diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index 5c6bf74f..00000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: CUDA (native + president-docker) - -run-name: >- - ${{ github.workflow }} on ${{ github.ref_name }} (#${{ github.run_number }}) - — ${{ github.event.pull_request.title || github.event.head_commit.message }} - -on: [ push ] - -permissions: - contents: read # checkout only - -# ────────────────────────────────────────────────────────────── -jobs: - # 1️⃣ Native matrix build (president + ttt) - build: - strategy: - fail-fast: false - matrix: - runnerLabel: [ president, ttt ] - - runs-on: - - self-hosted - - Linux - - X64 - - ${{ matrix.runnerLabel }} - - name: Native build on ${{ matrix.runnerLabel }} - - steps: - - uses: actions/checkout@v3 - with: { fetch-depth: 0 } - - - name: 🔎 Env dump - run: | - set -x - hostname - echo "Matrix label :" ${{ matrix.runnerLabel }} - which nvidia-smi || true - which nvcc || true - which cmake || true - - - name: ✅ Verify tools - run: | - for cmd in nvidia-smi nvcc cmake; do - command -v "$cmd" >/dev/null || { echo "::error::$cmd missing"; exit 127; } - done - nvidia-smi - nvcc --version - cmake --version - - - name: ⚙️ Configure - run: cmake -S . -B build -DFLS_BUILD_GPU=ON - - - name: 🛠️ Build - run: cmake --build build -- -j"$(nproc)" - - - name: 🚀 Smoke-test - run: ./build/cuda_info_dummy \ No newline at end of file From cb79bdaf2cc89568d877764fb5d1c68371351345 Mon Sep 17 00:00:00 2001 From: azimafroozeh Date: Tue, 22 Jul 2025 23:16:43 +0200 Subject: [PATCH 3/3] bump the seed --- test/src/quick_fuzz_tests/fuzz_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/quick_fuzz_tests/fuzz_config.json b/test/src/quick_fuzz_tests/fuzz_config.json index 0dc30769..80f49e46 100644 --- a/test/src/quick_fuzz_tests/fuzz_config.json +++ b/test/src/quick_fuzz_tests/fuzz_config.json @@ -1,6 +1,6 @@ { "num_cases": 10, - "base_seed": 2, + "base_seed": 3, "delimiter": "|", "min_cols": 1, "max_cols": 2,