From 66b20838111446bf67095c16d373cd6f2edf12b9 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 12:37:55 -0400 Subject: [PATCH 01/20] crypto: vendor pinned mlkem-native without source changes Vendor mlkem-native v2.0.0 (d1b2fe782888bdb761a50336012923180be7f502): the whole upstream mlkem/ directory and LICENSE, byte for byte, in src/mlkem-native. Nothing compiles it yet. contrib/devtools/mlkem-native.pin records the commit, tag, the tree id of mlkem/ and the LICENSE blob id. contrib/devtools/update-mlkem-native.sh reproduces the tree from upstream and rewrites the pin from the fetched object ids. test/lint/mlkem-native-check.sh, run by the subtree lint, proves offline that HEAD still holds exactly the pinned tree. Lints that scan src/ ignore the vendored tree, and doc/subtrees/mlkem-native.md is the runbook. Part of #184 (part 3, #190, D1). Co-Authored-By: Claude Opus 5.5 --- contrib/devtools/mlkem-native.pin | 8 + contrib/devtools/update-mlkem-native.sh | 147 + doc/subtrees/mlkem-native.md | 128 + src/mlkem-native/LICENSE | 312 ++ src/mlkem-native/mlkem/README.md | 23 + src/mlkem-native/mlkem/mlkem_native.c | 692 ++++ src/mlkem-native/mlkem/mlkem_native.h | 464 +++ src/mlkem-native/mlkem/mlkem_native_asm.S | 716 ++++ src/mlkem-native/mlkem/mlkem_native_config.h | 683 ++++ src/mlkem-native/mlkem/src/cbmc.h | 222 ++ src/mlkem-native/mlkem/src/common.h | 296 ++ src/mlkem-native/mlkem/src/compress.c | 763 ++++ src/mlkem-native/mlkem/src/compress.h | 613 ++++ src/mlkem-native/mlkem/src/context.h | 51 + src/mlkem-native/mlkem/src/debug.c | 64 + src/mlkem-native/mlkem/src/debug.h | 121 + src/mlkem-native/mlkem/src/fips202/fips202.c | 249 ++ src/mlkem-native/mlkem/src/fips202/fips202.h | 144 + .../mlkem/src/fips202/fips202x4.c | 207 ++ .../mlkem/src/fips202/fips202x4.h | 81 + .../mlkem/src/fips202/keccakf1600.c | 499 +++ .../mlkem/src/fips202/keccakf1600.h | 98 + .../mlkem/src/fips202/native/aarch64/auto.h | 78 + .../aarch64/src/fips202_native_aarch64.h | 80 + .../src/keccak_f1600_x1_scalar_aarch64_asm.S | 377 ++ .../src/keccak_f1600_x1_v84a_aarch64_asm.S | 206 ++ .../src/keccak_f1600_x2_v84a_aarch64_asm.S | 261 ++ ...k_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S | 1079 ++++++ ...00_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S | 989 +++++ .../aarch64/src/keccakf1600_round_constants.c | 47 + .../src/fips202/native/aarch64/x1_scalar.h | 26 + .../src/fips202/native/aarch64/x1_v84a.h | 35 + .../src/fips202/native/aarch64/x2_v84a.h | 38 + .../fips202/native/aarch64/x4_v8a_scalar.h | 31 + .../native/aarch64/x4_v8a_v84a_scalar.h | 36 + .../mlkem/src/fips202/native/api.h | 117 + .../src/fips202/native/armv81m/README.md | 10 + .../mlkem/src/fips202/native/armv81m/mve.h | 65 + .../armv81m/src/fips202_native_armv81m.h | 36 + .../native/armv81m/src/keccak_f1600_x4_mve.S | 716 ++++ .../native/armv81m/src/keccak_f1600_x4_mve.c | 40 + .../keccak_f1600_x4_state_extract_bytes_mve.S | 334 ++ .../src/keccak_f1600_x4_state_xor_bytes_mve.S | 355 ++ .../armv81m/src/keccakf1600_round_constants.c | 52 + .../mlkem/src/fips202/native/auto.h | 29 + .../native/x86_64/keccak_f1600_x4_avx2.h | 33 + .../native/x86_64/src/fips202_native_x86_64.h | 44 + .../x86_64/src/keccak_f1600_x4_avx2_asm.S | 487 +++ .../native/x86_64/src/keccakf1600_constants.c | 52 + src/mlkem-native/mlkem/src/indcpa.c | 675 ++++ src/mlkem-native/mlkem/src/indcpa.h | 172 + src/mlkem-native/mlkem/src/kem.c | 474 +++ src/mlkem-native/mlkem/src/kem.h | 353 ++ .../mlkem/src/native/aarch64/README.md | 16 + .../mlkem/src/native/aarch64/meta.h | 166 + .../src/native/aarch64/src/aarch64_zetas.c | 184 + .../native/aarch64/src/arith_native_aarch64.h | 184 + .../aarch64/src/mlkem_intt_aarch64_asm.S | 635 ++++ .../aarch64/src/mlkem_ntt_aarch64_asm.S | 565 +++ .../mlkem_poly_mulcache_compute_aarch64_asm.S | 130 + .../src/mlkem_poly_reduce_aarch64_asm.S | 153 + .../src/mlkem_poly_tobytes_aarch64_asm.S | 124 + .../src/mlkem_poly_tomont_aarch64_asm.S | 102 + ...mul_acc_montgomery_cached_k2_aarch64_asm.S | 264 ++ ...mul_acc_montgomery_cached_k3_aarch64_asm.S | 317 ++ ...mul_acc_montgomery_cached_k4_aarch64_asm.S | 371 ++ .../src/mlkem_rej_uniform_aarch64_asm.S | 226 ++ .../native/aarch64/src/rej_uniform_table.c | 543 +++ src/mlkem-native/mlkem/src/native/api.h | 651 ++++ src/mlkem-native/mlkem/src/native/meta.h | 30 + .../mlkem/src/native/ppc64le/README.md | 6 + .../mlkem/src/native/ppc64le/meta.h | 81 + .../native/ppc64le/src/arith_native_ppc64le.h | 24 + .../mlkem/src/native/ppc64le/src/consts.c | 299 ++ .../mlkem/src/native/ppc64le/src/consts.h | 34 + .../native/ppc64le/src/mlkem_intt_ppc_asm.S | 3246 +++++++++++++++++ .../native/ppc64le/src/mlkem_ntt_ppc_asm.S | 1672 +++++++++ .../ppc64le/src/mlkem_poly_tomont_ppc_asm.S | 316 ++ .../native/ppc64le/src/mlkem_reduce_ppc_asm.S | 731 ++++ .../mlkem/src/native/riscv64/README.md | 11 + .../mlkem/src/native/riscv64/meta.h | 137 + .../native/riscv64/src/arith_native_riscv64.h | 49 + .../src/native/riscv64/src/rv64v_debug.c | 83 + .../src/native/riscv64/src/rv64v_debug.h | 140 + .../src/native/riscv64/src/rv64v_izetas.inc | 27 + .../mlkem/src/native/riscv64/src/rv64v_poly.c | 797 ++++ .../src/native/riscv64/src/rv64v_zetas.inc | 27 + .../riscv64/src/rv64v_zetas_basemul.inc | 39 + .../mlkem/src/native/x86_64/README.md | 4 + .../mlkem/src/native/x86_64/meta.h | 324 ++ .../native/x86_64/src/arith_native_x86_64.h | 327 ++ .../src/native/x86_64/src/compress_consts.c | 115 + .../src/native/x86_64/src/compress_consts.h | 53 + .../mlkem/src/native/x86_64/src/consts.c | 102 + .../mlkem/src/native/x86_64/src/consts.h | 25 + .../native/x86_64/src/mlkem_intt_avx2_asm.S | 743 ++++ .../native/x86_64/src/mlkem_ntt_avx2_asm.S | 661 ++++ .../x86_64/src/mlkem_nttfrombytes_avx2_asm.S | 217 ++ .../x86_64/src/mlkem_ntttobytes_avx2_asm.S | 205 ++ .../x86_64/src/mlkem_nttunpack_avx2_asm.S | 190 + .../src/mlkem_poly_compress_d10_avx2_asm.S | 413 +++ .../src/mlkem_poly_compress_d11_avx2_asm.S | 479 +++ .../src/mlkem_poly_compress_d4_avx2_asm.S | 194 + .../src/mlkem_poly_compress_d5_avx2_asm.S | 251 ++ .../src/mlkem_poly_decompress_d10_avx2_asm.S | 257 ++ .../src/mlkem_poly_decompress_d11_avx2_asm.S | 307 ++ .../src/mlkem_poly_decompress_d4_avx2_asm.S | 209 ++ .../src/mlkem_poly_decompress_d5_avx2_asm.S | 222 ++ .../mlkem_poly_mulcache_compute_avx2_asm.S | 118 + ...asemul_acc_montgomery_cached_k2_avx2_asm.S | 536 +++ ...asemul_acc_montgomery_cached_k3_avx2_asm.S | 784 ++++ ...asemul_acc_montgomery_cached_k4_avx2_asm.S | 1032 ++++++ .../native/x86_64/src/mlkem_reduce_avx2_asm.S | 234 ++ .../x86_64/src/mlkem_rej_uniform_avx2_asm.S | 136 + .../native/x86_64/src/mlkem_tomont_avx2_asm.S | 172 + .../src/native/x86_64/src/rej_uniform_table.c | 545 +++ src/mlkem-native/mlkem/src/params.h | 76 + src/mlkem-native/mlkem/src/poly.c | 581 +++ src/mlkem-native/mlkem/src/poly.h | 296 ++ src/mlkem-native/mlkem/src/poly_k.c | 522 +++ src/mlkem-native/mlkem/src/poly_k.h | 622 ++++ src/mlkem-native/mlkem/src/randombytes.h | 52 + src/mlkem-native/mlkem/src/sampling.c | 371 ++ src/mlkem-native/mlkem/src/sampling.h | 111 + src/mlkem-native/mlkem/src/symmetric.h | 70 + src/mlkem-native/mlkem/src/sys.h | 320 ++ src/mlkem-native/mlkem/src/verify.c | 20 + src/mlkem-native/mlkem/src/verify.h | 447 +++ src/mlkem-native/mlkem/src/zetas.inc | 30 + test/lint/README.md | 16 + test/lint/lint-locale-dependence.py | 1 + test/lint/lint_ignore_dirs.py | 1 + test/lint/mlkem-native-check.sh | 128 + test/lint/test_runner/src/main.rs | 8 + 134 files changed, 38815 insertions(+) create mode 100644 contrib/devtools/mlkem-native.pin create mode 100755 contrib/devtools/update-mlkem-native.sh create mode 100644 doc/subtrees/mlkem-native.md create mode 100644 src/mlkem-native/LICENSE create mode 100644 src/mlkem-native/mlkem/README.md create mode 100644 src/mlkem-native/mlkem/mlkem_native.c create mode 100644 src/mlkem-native/mlkem/mlkem_native.h create mode 100644 src/mlkem-native/mlkem/mlkem_native_asm.S create mode 100644 src/mlkem-native/mlkem/mlkem_native_config.h create mode 100644 src/mlkem-native/mlkem/src/cbmc.h create mode 100644 src/mlkem-native/mlkem/src/common.h create mode 100644 src/mlkem-native/mlkem/src/compress.c create mode 100644 src/mlkem-native/mlkem/src/compress.h create mode 100644 src/mlkem-native/mlkem/src/context.h create mode 100644 src/mlkem-native/mlkem/src/debug.c create mode 100644 src/mlkem-native/mlkem/src/debug.h create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202.c create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202.h create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202x4.c create mode 100644 src/mlkem-native/mlkem/src/fips202/fips202x4.h create mode 100644 src/mlkem-native/mlkem/src/fips202/keccakf1600.c create mode 100644 src/mlkem-native/mlkem/src/fips202/keccakf1600.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/api.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c create mode 100644 src/mlkem-native/mlkem/src/fips202/native/auto.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c create mode 100644 src/mlkem-native/mlkem/src/indcpa.c create mode 100644 src/mlkem-native/mlkem/src/indcpa.h create mode 100644 src/mlkem-native/mlkem/src/kem.c create mode 100644 src/mlkem-native/mlkem/src/kem.h create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/README.md create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c create mode 100644 src/mlkem-native/mlkem/src/native/api.h create mode 100644 src/mlkem-native/mlkem/src/native/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/README.md create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/README.md create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc create mode 100644 src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/README.md create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/meta.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/consts.c create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/consts.h create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S create mode 100644 src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c create mode 100644 src/mlkem-native/mlkem/src/params.h create mode 100644 src/mlkem-native/mlkem/src/poly.c create mode 100644 src/mlkem-native/mlkem/src/poly.h create mode 100644 src/mlkem-native/mlkem/src/poly_k.c create mode 100644 src/mlkem-native/mlkem/src/poly_k.h create mode 100644 src/mlkem-native/mlkem/src/randombytes.h create mode 100644 src/mlkem-native/mlkem/src/sampling.c create mode 100644 src/mlkem-native/mlkem/src/sampling.h create mode 100644 src/mlkem-native/mlkem/src/symmetric.h create mode 100644 src/mlkem-native/mlkem/src/sys.h create mode 100644 src/mlkem-native/mlkem/src/verify.c create mode 100644 src/mlkem-native/mlkem/src/verify.h create mode 100644 src/mlkem-native/mlkem/src/zetas.inc create mode 100755 test/lint/mlkem-native-check.sh diff --git a/contrib/devtools/mlkem-native.pin b/contrib/devtools/mlkem-native.pin new file mode 100644 index 0000000000..5bef7bd4e1 --- /dev/null +++ b/contrib/devtools/mlkem-native.pin @@ -0,0 +1,8 @@ +# mlkem-native vendored into src/mlkem-native. Written by +# contrib/devtools/update-mlkem-native.sh; checked by +# test/lint/mlkem-native-check.sh. See doc/subtrees/mlkem-native.md. +url=https://github.com/pq-code-package/mlkem-native.git +tag=v2.0.0 +commit=d1b2fe782888bdb761a50336012923180be7f502 +mlkem_tree=d9d581290ea1e6fa37462bfbc61b9b4266a2e9e5 +license_blob=79c71b1fdcc53cc17b1df99a0f4d4d48121ac174 diff --git a/contrib/devtools/update-mlkem-native.sh b/contrib/devtools/update-mlkem-native.sh new file mode 100755 index 0000000000..20630919db --- /dev/null +++ b/contrib/devtools/update-mlkem-native.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +export LC_ALL=C +set -euo pipefail + +readonly PREFIX="src/mlkem-native" +readonly PIN_FILE="contrib/devtools/mlkem-native.pin" + +usage() { + cat <] + +Replace ${PREFIX} with the upstream mlkem/ directory and LICENSE of an +mlkem-native release, unmodified, rewrite ${PIN_FILE} from the fetched +upstream object ids, and stage both. + + tag or commit to vendor (default: REMOTE_REF, else the pinned tag) + +Environment overrides: + REMOTE_URL default: the url= entry of ${PIN_FILE} + REMOTE_REF default: the tag= entry of ${PIN_FILE} + +See doc/subtrees/mlkem-native.md. +EOF +} + +fail() { + local signature="$1" + local cause="$2" + local fix="$3" + echo "FAIL: ${signature}" >&2 + echo " Cause: ${cause}" >&2 + echo " Fix: ${fix}" >&2 + exit 1 +} + +pin_value() { + awk -F= -v key="$1" '$1 == key { print substr($0, length(key) + 2); exit }' "${PIN_FILE}" +} + +if [[ "${1-}" == "--help" || "${1-}" == "-h" ]]; then + usage + exit 0 +fi + +if [[ $# -gt 1 ]]; then + usage >&2 + exit 2 +fi + +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + fail "not inside a git work tree" \ + "the update stages the vendored tree with git" \ + "run it from the root of a qbit checkout" +fi +cd "$(git rev-parse --show-toplevel)" + +pinned_url="$(pin_value url)" +pinned_tag="$(pin_value tag)" +pinned_commit="$(pin_value commit)" + +REMOTE_URL="${REMOTE_URL:-${pinned_url}}" +REMOTE_REF="${1:-${REMOTE_REF:-${pinned_tag}}}" +if [[ -z "${REMOTE_URL}" || -z "${REMOTE_REF}" ]]; then + fail "no upstream url or ref" \ + "${PIN_FILE} lacks url= or tag= and no override was given" \ + "restore ${PIN_FILE} from git, or set REMOTE_URL and pass a tag" +fi + +echo "Fetching mlkem-native ${REMOTE_REF} from ${REMOTE_URL}" +if ! git fetch --quiet --depth=1 --no-tags "${REMOTE_URL}" "${REMOTE_REF}"; then + fail "git fetch ${REMOTE_URL} ${REMOTE_REF} failed" \ + "the ref does not exist upstream, or the network is unavailable" \ + "check the tag name at ${REMOTE_URL}, or retry with network access" +fi +commit="$(git rev-parse "FETCH_HEAD^{commit}")" + +# A tag that resolves to a different commit than the pin means upstream moved +# it; never follow that silently. +if [[ "${REMOTE_REF}" == "${pinned_tag}" && "${commit}" != "${pinned_commit}" ]]; then + fail "${REMOTE_REF} resolves to ${commit}, but ${PIN_FILE} pins ${pinned_commit}" \ + "the upstream tag was moved or ${REMOTE_URL} is not the pinned repository" \ + "stop and investigate upstream; to vendor a different commit deliberately, pass it explicitly" +fi + +tag="${REMOTE_REF}" +if [[ "${REMOTE_REF}" =~ ^[0-9a-f]{40}$ ]]; then + tag="$(git ls-remote --tags "${REMOTE_URL}" | + awk -v c="${commit}" '$1 == c { sub("^refs/tags/", "", $2); sub("\\^\\{\\}$", "", $2); print $2; exit }')" + if [[ -z "${tag}" ]]; then + echo "warning: ${commit} has no upstream tag; releases must vendor a tagged commit" >&2 + tag="untagged" + fi +fi + +upstream_tree="$(git rev-parse "${commit}:mlkem")" +upstream_license="$(git rev-parse "${commit}:LICENSE")" + +echo "Replacing ${PREFIX} with ${commit}:mlkem and ${commit}:LICENSE" +rm -rf -- "${PREFIX}" +mkdir -p -- "${PREFIX}" +git archive --format=tar "${commit}" mlkem LICENSE | tar -xf - -C "${PREFIX}" + +cat > "${PIN_FILE}" < differs from pinned tree ` + (or the same for `LICENSE`) + Cause: + A vendored file was edited, added, removed or changed mode, or the pin was + changed without re-vendoring. + Fix: + Never edit `src/mlkem-native`. Restore it with + `contrib/devtools/update-mlkem-native.sh`. Changes belong upstream, or in + qbit's glue outside the vendored directory. + +2. Signature: + `FAIL: uncommitted changes under src/mlkem-native` + Cause: + The work tree or index has edits, new files or deletions under the vendored + directory. + Fix: + `git checkout HEAD -- src/mlkem-native && git clean -fdx -- src/mlkem-native`. + +3. Signature: + `FAIL: src/mlkem-native in HEAD holds entries other than LICENSE and mlkem` + Cause: + A qbit file was added next to the vendored tree. + Fix: + Move it out; qbit's ML-KEM glue lives in `src/crypto/mlkem*`. + +4. Signature: + `FAIL: resolves to , but contrib/devtools/mlkem-native.pin pins ` + Cause: + Upstream moved the tag, or `REMOTE_URL` points at a different repository. + Fix: + Stop and investigate upstream before vendoring anything. To vendor a + different commit deliberately, pass it to the script explicitly. + +5. Signature: + `FAIL: staged src/mlkem-native (...) differs from upstream (...)` + Cause: + A git attribute, filter or line-ending setting (for example + `core.autocrlf`) rewrote vendored bytes on staging. + Fix: + Remove the setting that applies to `src/mlkem-native` and re-run the update + script. + +## Review Cadence + +At least quarterly, check +[upstream releases](https://github.com/pq-code-package/mlkem-native/releases) +and [security advisories](https://github.com/pq-code-package/mlkem-native/security/advisories). +Update promptly for a security fix; otherwise update when a release brings a +proved improvement qbit uses. diff --git a/src/mlkem-native/LICENSE b/src/mlkem-native/LICENSE new file mode 100644 index 0000000000..79c71b1fdc --- /dev/null +++ b/src/mlkem-native/LICENSE @@ -0,0 +1,312 @@ +mlkem-native is a fork of the public domain Kyber reference +implementation, available on https://github.com/pq-crystals/kyber. + +All source code in mlkem/* and dev/* is licensed under your choice +of the Apache-2.0 license OR the ISC license OR the MIT license. +These licenses are reproduced at the bottom of this file. +The copyright holders are indicated at the top of each file. + +Files outside the library itself may carry different terms. Every file +states its own SPDX-License-Identifier, which determines the terms that +apply to it. In particular: + +The code in test/notrandombytes/*, and its copies in +examples/*/test_only_rng/* and scripts/notrandombytes, is derived from +https://cr.yp.to/papers.html#surf and licensed under +LicenseRef-PD-hp OR CC0-1.0 OR 0BSD OR MIT-0 OR MIT. +It is only used for testing purposes. + +The benchmarking code in test/hal/* carries the +MIT license. It is only used for testing purposes. + +The tiny_sha3 code in +examples/bring_your_own_fips202/custom_fips202/tiny_sha3/* and +examples/custom_backend/mlkem_native/src/fips202/native/custom/src/* +carries the MIT license. It is only used to demonstrate custom +FIPS-202 implementations. + +The proofs in proofs/* are in part derived from Amazon Web Services +verification infrastructure. Individual files carry, among others, +Apache-2.0 OR ISC OR MIT-0, MIT-0, and MIT-0 AND Apache-2.0. None of the +proofs are part of the library. + +Documentation is licensed under CC-BY-4.0. + +``` +Copyright (c) The mlkem-native project authors +Copyright (c) 2020 Dougall Johnson +Copyright (c) 2022 Arm Limited +SPDX-License-Identifier: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +The tiny_sha3 implementation in examples/custom_backend/mlkem_native/src/fips202/native/custom/src/ +carries the MIT license. It is only used for testing purposes. + +``` +SPDX-License-Identifier: MIT + +19-Nov-11 Markku-Juhani O. Saarinen +``` + +ISC license +----------- + +Copyright + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +MIT license +----------- + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Apache-2.0 license +------------------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/mlkem-native/mlkem/README.md b/src/mlkem-native/mlkem/README.md new file mode 100644 index 0000000000..e648c436a5 --- /dev/null +++ b/src/mlkem-native/mlkem/README.md @@ -0,0 +1,23 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# mlkem-native source tree + +This is the main source tree of mlkem-native. + +## Building + +To build a mlkem-native for a fixed parameter set (ML-KEM-512/768/1024), build the compilation in units in `src/*` separately, and link to an RNG and your application. See [examples/basic](../examples/basic) for a simple example. + +Alternatively, you can use the auto-generated helper files [mlkem_native.c](mlkem_native.c) and [mlkem_native_asm.S](mlkem_native_asm.S), which bundle all *.c and *.S files together. See [examples/monolithic_build](../examples/monolithic_build) and [examples/monolithic_build_native](../examples/monolithic_build_native) for examples without and with native code. + +## Configuration + +The build is configured by [mlkem_native_config.h](mlkem_native_config.h), or by the file pointed to by `MLK_CONFIG_FILE`. Note in particular `MLK_CONFIG_PARAMETER_SET` and `MLK_CONFIG_NAMESPACE_PREFIX`, which set the parameter set and namespace prefix, respectively. + +## API + +The public API is defined in [mlkem_native.h](mlkem_native.h). + +## Supporting multiple parameter sets + +If you want to support multiple parameter sets, build the library once per parameter set you want to support. Set `MLK_CONFIG_MULTILEVEL_BUILD` for all of these builds; this is what appends the parameter set to the namespace prefix, keeping the level-specific symbols apart. To avoid duplicating shared functionality, additionally set `MLK_CONFIG_MULTILEVEL_WITH_SHARED` for exactly one of the builds, and `MLK_CONFIG_MULTILEVEL_NO_SHARED` for the others. Finally, link with RNG and your application as before. This is demonstrated in the examples [examples/multilevel_build](../examples/multilevel_build), [examples/multilevel_build_native](../examples/multilevel_build_native), [examples/monolithic_build_multilevel](../examples/monolithic_build_multilevel) and [examples/monolithic_build_multilevel_native](../examples/monolithic_build_multilevel_native). diff --git a/src/mlkem-native/mlkem/mlkem_native.c b/src/mlkem-native/mlkem/mlkem_native.c new file mode 100644 index 0000000000..66aed07ece --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native.c @@ -0,0 +1,692 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +/****************************************************************************** + * + * Single compilation unit (SCU) for fixed-level build of mlkem-native + * + * This compilation unit bundles together all source files for a build + * of mlkem-native for a fixed security level (MLKEM-512/768/1024). + * + * # API + * + * The API exposed by this file is described in mlkem_native.h. + * + * # Multi-level build + * + * If you want an SCU build of mlkem-native with support for multiple security + * levels, you need to include this file multiple times, and set + * MLK_CONFIG_MULTILEVEL_WITH_SHARED and MLK_CONFIG_MULTILEVEL_NO_SHARED + * appropriately. This is exemplified in examples/monolithic_build_multilevel + * and examples/monolithic_build_multilevel_native. + * + * # Configuration + * + * The following options from the mlkem-native configuration are relevant: + * + * - MLK_CONFIG_FIPS202_CUSTOM_HEADER + * Set this option if you use a custom FIPS202 implementation. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * Set this option if you want to include the native arithmetic backends + * in your build. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * Set this option if you want to include the native FIPS202 backends + * in your build. + * + * - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * Set this option if you want to keep the directives defined in + * level-independent headers. This is needed for a multi-level build. + */ + +/* If parts of the mlkem-native source tree are not used, + * consider reducing this header via `unifdef`. + * + * Example: + * ```bash + * unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native.c + * ``` + */ + +#include "src/common.h" + +#include "src/compress.c" +#include "src/debug.c" +#include "src/indcpa.c" +#include "src/kem.c" +#include "src/poly.c" +#include "src/poly_k.c" +#include "src/sampling.c" +#include "src/verify.c" + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +#include "src/fips202/fips202.c" +#include "src/fips202/fips202x4.c" +#include "src/fips202/keccakf1600.c" +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#if defined(MLK_SYS_AARCH64) +#include "src/native/aarch64/src/aarch64_zetas.c" +#include "src/native/aarch64/src/rej_uniform_table.c" +#endif +#if defined(MLK_SYS_X86_64) +#include "src/native/x86_64/src/compress_consts.c" +#include "src/native/x86_64/src/consts.c" +#include "src/native/x86_64/src/rej_uniform_table.c" +#endif +#if defined(MLK_SYS_RISCV64) +#include "src/native/riscv64/src/rv64v_debug.c" +#include "src/native/riscv64/src/rv64v_poly.c" +#endif +#if defined(MLK_SYS_PPC64LE) +#include "src/native/ppc64le/src/consts.c" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#if defined(MLK_SYS_AARCH64) +#include "src/fips202/native/aarch64/src/keccakf1600_round_constants.c" +#endif +#if defined(MLK_SYS_X86_64) +#include "src/fips202/native/x86_64/src/keccakf1600_constants.c" +#endif +#if defined(MLK_SYS_ARMV81M_MVE) +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c" +#include "src/fips202/native/armv81m/src/keccakf1600_round_constants.c" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + +/* Macro #undef's + * + * The following undefines macros from headers + * included by the source files imported above. + * + * This is to allow building and linking multiple builds + * of mlkem-native for varying parameter sets through concatenation + * of this file, as if the files had been compiled separately. + * If this is not relevant to you, you may remove the following. + */ + +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files + */ +/* mlkem/mlkem_native.h */ +#undef MLKEM1024_BYTES +#undef MLKEM1024_CIPHERTEXTBYTES +#undef MLKEM1024_PUBLICKEYBYTES +#undef MLKEM1024_SECRETKEYBYTES +#undef MLKEM1024_SYMBYTES +#undef MLKEM512_BYTES +#undef MLKEM512_CIPHERTEXTBYTES +#undef MLKEM512_PUBLICKEYBYTES +#undef MLKEM512_SECRETKEYBYTES +#undef MLKEM512_SYMBYTES +#undef MLKEM768_BYTES +#undef MLKEM768_CIPHERTEXTBYTES +#undef MLKEM768_PUBLICKEYBYTES +#undef MLKEM768_SECRETKEYBYTES +#undef MLKEM768_SYMBYTES +#undef MLKEM_BYTES +#undef MLKEM_CIPHERTEXTBYTES +#undef MLKEM_CIPHERTEXTBYTES_ +#undef MLKEM_PUBLICKEYBYTES +#undef MLKEM_PUBLICKEYBYTES_ +#undef MLKEM_SECRETKEYBYTES +#undef MLKEM_SECRETKEYBYTES_ +#undef MLKEM_SYMBYTES +#undef MLK_API_CONCAT +#undef MLK_API_CONCAT_ +#undef MLK_API_CONCAT_UNDERSCORE +#undef MLK_API_MUST_CHECK_RETURN_VALUE +#undef MLK_API_NAMESPACE +#undef MLK_API_NAMESPACE_PREFIX +#undef MLK_API_QUALIFIER +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_H +#undef MLK_MAX3_ +#undef MLK_TOTAL_ALLOC_1024 +#undef MLK_TOTAL_ALLOC_1024_DECAPS +#undef MLK_TOTAL_ALLOC_1024_ENCAPS +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_512 +#undef MLK_TOTAL_ALLOC_512_DECAPS +#undef MLK_TOTAL_ALLOC_512_ENCAPS +#undef MLK_TOTAL_ALLOC_512_KEYPAIR +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_768 +#undef MLK_TOTAL_ALLOC_768_DECAPS +#undef MLK_TOTAL_ALLOC_768_ENCAPS +#undef MLK_TOTAL_ALLOC_768_KEYPAIR +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +/* mlkem/src/common.h */ +#undef MLK_ADD_PARAM_SET +#undef MLK_ALLOC +#undef MLK_APPLY +#undef MLK_ASM_FN_SIZE +#undef MLK_ASM_FN_SYMBOL +#undef MLK_ASM_NAMESPACE +#undef MLK_BUILD_INTERNAL +#undef MLK_COMMON_H +#undef MLK_CONCAT +#undef MLK_CONCAT_ +#undef MLK_EMPTY_CU +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_EXTERNAL_API +#undef MLK_FIPS202X4_HEADER_FILE +#undef MLK_FIPS202_HEADER_FILE +#undef MLK_FREE +#undef MLK_INTERNAL_API +#undef MLK_INTERNAL_DATA_DECLARATION +#undef MLK_INTERNAL_DATA_DEFINITION +#undef MLK_NAMESPACE +#undef MLK_NAMESPACE_K +#undef MLK_NAMESPACE_PREFIX +#undef MLK_NAMESPACE_PREFIX_K +#undef mlk_memcpy +#undef mlk_memset +/* mlkem/src/indcpa.h */ +#undef MLK_INDCPA_H +#undef mlk_gen_matrix +#undef mlk_indcpa_dec +#undef mlk_indcpa_enc +#undef mlk_indcpa_keypair_derand +/* mlkem/src/kem.h */ +#undef MLK_KEM_H +#undef mlk_kem_check_pk +#undef mlk_kem_check_sk +#undef mlk_kem_dec +#undef mlk_kem_enc +#undef mlk_kem_enc_derand +#undef mlk_kem_keypair +#undef mlk_kem_keypair_derand +/* mlkem/src/params.h */ +#undef MLKEM_DU +#undef MLKEM_DV +#undef MLKEM_ETA1 +#undef MLKEM_ETA2 +#undef MLKEM_INDCCA_CIPHERTEXTBYTES +#undef MLKEM_INDCCA_PUBLICKEYBYTES +#undef MLKEM_INDCCA_SECRETKEYBYTES +#undef MLKEM_INDCPA_BYTES +#undef MLKEM_INDCPA_MSGBYTES +#undef MLKEM_INDCPA_PUBLICKEYBYTES +#undef MLKEM_INDCPA_SECRETKEYBYTES +#undef MLKEM_K +#undef MLKEM_N +#undef MLKEM_POLYBYTES +#undef MLKEM_POLYCOMPRESSEDBYTES_D10 +#undef MLKEM_POLYCOMPRESSEDBYTES_D11 +#undef MLKEM_POLYCOMPRESSEDBYTES_D4 +#undef MLKEM_POLYCOMPRESSEDBYTES_D5 +#undef MLKEM_POLYCOMPRESSEDBYTES_DU +#undef MLKEM_POLYCOMPRESSEDBYTES_DV +#undef MLKEM_POLYVECBYTES +#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU +#undef MLKEM_Q +#undef MLKEM_Q_HALF +#undef MLKEM_SSBYTES +#undef MLKEM_SYMBYTES +#undef MLKEM_UINT12_LIMIT +#undef MLK_PARAMS_H +/* mlkem/src/poly_k.h */ +#undef MLK_POLY_K_H +#undef mlk_poly_compress_du +#undef mlk_poly_compress_dv +#undef mlk_poly_decompress_du +#undef mlk_poly_decompress_dv +#undef mlk_poly_getnoise_eta1122_4x +#undef mlk_poly_getnoise_eta1_4x +#undef mlk_poly_getnoise_eta2 +#undef mlk_poly_getnoise_eta2_4x +#undef mlk_polymat +#undef mlk_polyvec +#undef mlk_polyvec_add +#undef mlk_polyvec_basemul_acc_montgomery_cached +#undef mlk_polyvec_compress_du +#undef mlk_polyvec_decompress_du +#undef mlk_polyvec_frombytes +#undef mlk_polyvec_invntt_tomont +#undef mlk_polyvec_mulcache +#undef mlk_polyvec_mulcache_compute +#undef mlk_polyvec_ntt +#undef mlk_polyvec_reduce +#undef mlk_polyvec_tobytes +#undef mlk_polyvec_tomont + +#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS) +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files + */ +/* mlkem/src/compress.h */ +#undef MLK_COMPRESS_H +#undef mlk_poly_compress_d10 +#undef mlk_poly_compress_d11 +#undef mlk_poly_compress_d4 +#undef mlk_poly_compress_d5 +#undef mlk_poly_decompress_d10 +#undef mlk_poly_decompress_d11 +#undef mlk_poly_decompress_d4 +#undef mlk_poly_decompress_d5 +#undef mlk_poly_frombytes +#undef mlk_poly_frommsg +#undef mlk_poly_tobytes +#undef mlk_poly_tomsg +/* mlkem/src/context.h */ +#undef MLK_CONTEXT_H +#undef MLK_CONTEXT_PARAMETERS_0 +#undef MLK_CONTEXT_PARAMETERS_1 +#undef MLK_CONTEXT_PARAMETERS_2 +#undef MLK_CONTEXT_PARAMETERS_3 +#undef MLK_CONTEXT_PARAMETERS_4 +#undef MLK_CONTEXT_UNUSED +/* mlkem/src/debug.h */ +#undef MLK_DEBUG_H +#undef mlk_assert +#undef mlk_assert_abs_bound +#undef mlk_assert_abs_bound_2d +#undef mlk_assert_bound +#undef mlk_assert_bound_2d +#undef mlk_debug_check_assert +#undef mlk_debug_check_bounds +/* mlkem/src/poly.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NTT_BOUND +#undef MLK_POLY_H +#undef mlk_poly_add +#undef mlk_poly_invntt_tomont +#undef mlk_poly_mulcache_compute +#undef mlk_poly_ntt +#undef mlk_poly_reduce +#undef mlk_poly_sub +#undef mlk_poly_tomont +/* mlkem/src/randombytes.h */ +#undef MLK_RANDOMBYTES_H +/* mlkem/src/sampling.h */ +#undef MLK_SAMPLING_H +#undef mlk_poly_cbd2 +#undef mlk_poly_cbd3 +#undef mlk_poly_rej_uniform +#undef mlk_poly_rej_uniform_x4 +/* mlkem/src/symmetric.h */ +#undef MLK_SYMMETRIC_H +#undef MLK_XOF_RATE +#undef mlk_hash_g +#undef mlk_hash_h +#undef mlk_hash_j +#undef mlk_prf_eta +#undef mlk_prf_eta1 +#undef mlk_prf_eta1_x4 +#undef mlk_prf_eta2 +#undef mlk_xof_absorb +#undef mlk_xof_ctx +#undef mlk_xof_init +#undef mlk_xof_release +#undef mlk_xof_squeezeblocks +#undef mlk_xof_x4_absorb +#undef mlk_xof_x4_ctx +#undef mlk_xof_x4_init +#undef mlk_xof_x4_release +#undef mlk_xof_x4_squeezeblocks +/* mlkem/src/sys.h */ +#undef MLK_ALIGN +#undef MLK_ALIGN_UP +#undef MLK_ALWAYS_INLINE +#undef MLK_CET_ENDBR +#undef MLK_CT_TESTING_DECLASSIFY +#undef MLK_CT_TESTING_SECRET +#undef MLK_DEFAULT_ALIGN +#undef MLK_HAVE_INLINE_ASM +#undef MLK_INLINE +#undef MLK_MUST_CHECK_RETURN_VALUE +#undef MLK_NOINLINE +#undef MLK_RESTRICT +#undef MLK_STATIC_TESTABLE +#undef MLK_SYSV_ABI +#undef MLK_SYSV_ABI_SUPPORTED +#undef MLK_SYS_AARCH64 +#undef MLK_SYS_AARCH64_EB +#undef MLK_SYS_AARCH64_NEON +#undef MLK_SYS_APPLE +#undef MLK_SYS_ARMV81M_MVE +#undef MLK_SYS_BIG_ENDIAN +#undef MLK_SYS_H +#undef MLK_SYS_LINUX +#undef MLK_SYS_LITTLE_ENDIAN +#undef MLK_SYS_PPC64LE +#undef MLK_SYS_RISCV32 +#undef MLK_SYS_RISCV64 +#undef MLK_SYS_RISCV64_RVV +#undef MLK_SYS_WINDOWS +#undef MLK_SYS_X86_64 +#undef MLK_SYS_X86_64_AVX2 +/* mlkem/src/verify.h */ +#undef MLK_USE_ASM_VALUE_BARRIER +#undef MLK_VERIFY_H +#undef mlk_ct_opt_blocker_u64 +/* mlkem/src/cbmc.h */ +#undef MLK_CBMC_H +#undef __contract__ +#undef __loop__ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +/* + * Undefine macros from FIPS-202 files + */ +/* mlkem/src/fips202/fips202.h */ +#undef FIPS202_X4_DEFAULT_IMPLEMENTATION +#undef MLK_FIPS202_FIPS202_H +#undef SHA3_256_HASHBYTES +#undef SHA3_256_RATE +#undef SHA3_384_RATE +#undef SHA3_512_HASHBYTES +#undef SHA3_512_RATE +#undef SHAKE128_RATE +#undef SHAKE256_RATE +#undef mlk_sha3_256 +#undef mlk_sha3_512 +#undef mlk_shake128_absorb_once +#undef mlk_shake128_init +#undef mlk_shake128_release +#undef mlk_shake128_squeezeblocks +#undef mlk_shake256 +/* mlkem/src/fips202/fips202x4.h */ +#undef MLK_FIPS202_FIPS202X4_H +#undef mlk_shake128x4_absorb_once +#undef mlk_shake128x4_init +#undef mlk_shake128x4_release +#undef mlk_shake128x4_squeezeblocks +#undef mlk_shake256x4 +/* mlkem/src/fips202/keccakf1600.h */ +#undef MLK_FIPS202_KECCAKF1600_H +#undef MLK_KECCAK_LANES +#undef MLK_KECCAK_WAY +#undef mlk_keccakf1600_extract_bytes +#undef mlk_keccakf1600_permute +#undef mlk_keccakf1600_xor_bytes +#undef mlk_keccakf1600x4_extract_bytes +#undef mlk_keccakf1600x4_permute +#undef mlk_keccakf1600x4_xor_bytes +#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* mlkem/src/fips202/native/api.h */ +#undef MLK_FIPS202_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +/* mlkem/src/fips202/native/auto.h */ +#undef MLK_FIPS202_NATIVE_AUTO_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (FIPS202, AArch64) + */ +/* mlkem/src/fips202/native/aarch64/auto.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#undef mlk_keccak_f1600_x1_scalar_aarch64_asm +#undef mlk_keccak_f1600_x1_v84a_aarch64_asm +#undef mlk_keccak_f1600_x2_v84a_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm +#undef mlk_keccakf1600_round_constants +/* mlkem/src/fips202/native/aarch64/x1_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x1_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x2_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X2_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (FIPS202, x86_64) + */ +/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */ +#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#undef MLK_FIPS202_X86_64_NEED_X4_AVX2 +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */ +#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#undef mlk_keccak_f1600_x4_avx2_asm +#undef mlk_keccak_rho56 +#undef mlk_keccak_rho8 +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_ARMV81M_MVE) +/* + * Undefine macros from native code (FIPS202, Armv8.1-M) + */ +/* mlkem/src/fips202/native/armv81m/mve.h */ +#undef MLK_FIPS202_ARMV81M_NEED_X4 +#undef MLK_FIPS202_NATIVE_ARMV81M +#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#undef MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES +#undef MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES +#undef mlk_keccak_f1600_x4_native_impl +/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */ +#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#undef mlk_keccak_f1600_x4_mve_asm +#undef mlk_keccak_f1600_x4_state_extract_bytes_asm +#undef mlk_keccak_f1600_x4_state_xor_bytes_asm +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_ARMV81M_MVE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* mlkem/src/native/api.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +#undef MLK_NTT_BOUND +/* mlkem/src/native/meta.h */ +#undef MLK_NATIVE_META_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (Arith, AArch64) + */ +/* mlkem/src/native/aarch64/meta.h */ +#undef MLK_ARITH_BACKEND_AARCH64 +#undef MLK_NATIVE_AARCH64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */ +#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#undef mlk_aarch64_invntt_zetas_layer12345 +#undef mlk_aarch64_invntt_zetas_layer67 +#undef mlk_aarch64_ntt_zetas_layer12345 +#undef mlk_aarch64_ntt_zetas_layer67 +#undef mlk_aarch64_zetas_mulcache_native +#undef mlk_aarch64_zetas_mulcache_twisted_native +#undef mlk_intt_aarch64_asm +#undef mlk_ntt_aarch64_asm +#undef mlk_poly_mulcache_compute_aarch64_asm +#undef mlk_poly_reduce_aarch64_asm +#undef mlk_poly_tobytes_aarch64_asm +#undef mlk_poly_tomont_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm +#undef mlk_rej_uniform_aarch64_asm +#undef mlk_rej_uniform_table +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (Arith, X86_64) + */ +/* mlkem/src/native/x86_64/meta.h */ +#undef MLK_ARITH_BACKEND_X86_64_DEFAULT +#undef MLK_NATIVE_X86_64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_COMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_FROMBYTES +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */ +#undef MLK_AVX2_REJ_UNIFORM_BUFLEN +#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#undef mlk_invntt_avx2_asm +#undef mlk_ntt_avx2_asm +#undef mlk_nttfrombytes_avx2_asm +#undef mlk_ntttobytes_avx2_asm +#undef mlk_nttunpack_avx2_asm +#undef mlk_poly_compress_d10_avx2_asm +#undef mlk_poly_compress_d11_avx2_asm +#undef mlk_poly_compress_d4_avx2_asm +#undef mlk_poly_compress_d5_avx2_asm +#undef mlk_poly_decompress_d10_avx2_asm +#undef mlk_poly_decompress_d11_avx2_asm +#undef mlk_poly_decompress_d4_avx2_asm +#undef mlk_poly_decompress_d5_avx2_asm +#undef mlk_poly_mulcache_compute_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm +#undef mlk_reduce_avx2_asm +#undef mlk_rej_uniform_avx2_asm +#undef mlk_rej_uniform_table +#undef mlk_tomont_avx2_asm +/* mlkem/src/native/x86_64/src/compress_consts.h */ +#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#undef mlk_compress_d10_data +#undef mlk_compress_d11_data +#undef mlk_compress_d4_data +#undef mlk_compress_d5_data +#undef mlk_decompress_d10_data +#undef mlk_decompress_d11_data +#undef mlk_decompress_d4_data +#undef mlk_decompress_d5_data +/* mlkem/src/native/x86_64/src/consts.h */ +#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD +#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP +#undef MLK_NATIVE_X86_64_SRC_CONSTS_H +#undef mlk_qdata +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mlkem/src/native/riscv64/meta.h */ +#undef MLK_ARITH_BACKEND_RISCV64 +#undef MLK_NATIVE_RISCV64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mlk_rv64v_poly_add +#undef mlk_rv64v_poly_basemul_mont_add_k2 +#undef mlk_rv64v_poly_basemul_mont_add_k3 +#undef mlk_rv64v_poly_basemul_mont_add_k4 +#undef mlk_rv64v_poly_invntt_tomont +#undef mlk_rv64v_poly_ntt +#undef mlk_rv64v_poly_reduce +#undef mlk_rv64v_poly_sub +#undef mlk_rv64v_poly_tomont +#undef mlk_rv64v_rej_uniform +/* mlkem/src/native/riscv64/src/rv64v_debug.h */ +#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#undef mlk_assert_abs_bound_int16m1 +#undef mlk_assert_abs_bound_int16m2 +#undef mlk_assert_bound_int16m1 +#undef mlk_assert_bound_int16m2 +#undef mlk_debug_check_bounds_int16m1 +#undef mlk_debug_check_bounds_int16m2 +#endif /* MLK_SYS_RISCV64 */ +#if defined(MLK_SYS_PPC64LE) +/* + * Undefine macros from native code (Arith, PPC64LE) + */ +/* mlkem/src/native/ppc64le/meta.h */ +#undef MLK_ARITH_BACKEND_NAME +#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT +#undef MLK_NATIVE_PPC64LE_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */ +#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#undef mlk_intt_ppc_asm +#undef mlk_ntt_ppc_asm +#undef mlk_poly_tomont_ppc_asm +#undef mlk_reduce_ppc_asm +/* mlkem/src/native/ppc64le/src/consts.h */ +#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#undef MLK_PPC_C20159_OFFSET +#undef MLK_PPC_NQ_OFFSET +#undef MLK_PPC_N_INV_OFFSET +#undef MLK_PPC_N_INV_TW_OFFSET +#undef MLK_PPC_Q_OFFSET +#undef MLK_PPC_TOMONT_OFFSET +#undef MLK_PPC_TOMONT_TW_OFFSET +#undef MLK_PPC_ZETA_INTT_OFFSET +#undef MLK_PPC_ZETA_INTT_TW_OFFSET +#undef MLK_PPC_ZETA_NTT_OFFSET +#undef MLK_PPC_ZETA_NTT_TW_OFFSET +#undef mlk_ppc_qdata +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ diff --git a/src/mlkem-native/mlkem/mlkem_native.h b/src/mlkem-native/mlkem/mlkem_native.h new file mode 100644 index 0000000000..fe8060aa8d --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native.h @@ -0,0 +1,464 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_H +#define MLK_H + +/* + * Public API for mlkem-native. + * + * This header defines the public API of a single build of mlkem-native. + * + * Make sure the configuration file is in the include path + * (this is "mlkem_native_config.h" by default, or MLK_CONFIG_FILE if defined). + * + * # API conventions + * + * Conventions shared by all functions below (return values, pointer validity, + * output buffers on error) are documented in API-CONVENTIONS.md. + * + * # Multi-level builds + * + * This header specifies a build of mlkem-native for a fixed security level. + * If you need multiple security levels, leave the security level unspecified + * in the configuration file and include this header multiple times, setting + * MLK_CONFIG_PARAMETER_SET accordingly for each, and #undef'ing the MLK_H + * guard to allow multiple inclusions. + * + * In this case, the configuration file must also set + * MLK_CONFIG_MULTILEVEL_BUILD. Without it, the parameter set is not appended + * to the namespace prefix and all inclusions declare the same symbol names. + */ + +/******************************* Key sizes ************************************/ + +/* Sizes of cryptographic material, per parameter set */ +/* See mlkem/src/params.h for the arithmetic expressions giving rise to these */ +/* check-magic: off */ +#define MLKEM512_SECRETKEYBYTES 1632 +#define MLKEM512_PUBLICKEYBYTES 800 +#define MLKEM512_CIPHERTEXTBYTES 768 + +#define MLKEM768_SECRETKEYBYTES 2400 +#define MLKEM768_PUBLICKEYBYTES 1184 +#define MLKEM768_CIPHERTEXTBYTES 1088 + +#define MLKEM1024_SECRETKEYBYTES 3168 +#define MLKEM1024_PUBLICKEYBYTES 1568 +#define MLKEM1024_CIPHERTEXTBYTES 1568 +/* check-magic: on */ + +/* Size of randomness coins in bytes (level-independent) */ +#define MLKEM_SYMBYTES 32 +#define MLKEM512_SYMBYTES MLKEM_SYMBYTES +#define MLKEM768_SYMBYTES MLKEM_SYMBYTES +#define MLKEM1024_SYMBYTES MLKEM_SYMBYTES +/* Size of shared secret in bytes (level-independent) */ +#define MLKEM_BYTES 32 +#define MLKEM512_BYTES MLKEM_BYTES +#define MLKEM768_BYTES MLKEM_BYTES +#define MLKEM1024_BYTES MLKEM_BYTES + +/* Sizes of cryptographic material, as a function of LVL=512,768,1024 */ +#define MLKEM_SECRETKEYBYTES_(LVL) MLKEM##LVL##_SECRETKEYBYTES +#define MLKEM_PUBLICKEYBYTES_(LVL) MLKEM##LVL##_PUBLICKEYBYTES +#define MLKEM_CIPHERTEXTBYTES_(LVL) MLKEM##LVL##_CIPHERTEXTBYTES +#define MLKEM_SECRETKEYBYTES(LVL) MLKEM_SECRETKEYBYTES_(LVL) +#define MLKEM_PUBLICKEYBYTES(LVL) MLKEM_PUBLICKEYBYTES_(LVL) +#define MLKEM_CIPHERTEXTBYTES(LVL) MLKEM_CIPHERTEXTBYTES_(LVL) + +/****************************** Error codes ***********************************/ + +/* Generic failure condition. Currently not returned by any function; + * reserved for failures that no more specific code covers. */ +#define MLK_ERR_FAIL (-1) +/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is defined and the provided MLK_CUSTOM_ALLOC can fail. */ +#define MLK_ERR_OUT_OF_MEMORY (-2) +/* An RNG failure occurred. Might be due to insufficient entropy or + * system misconfiguration. */ +#define MLK_ERR_RNG_FAIL (-3) +/* Public key validation failed: the @[FIPS203, Section 7.2, 'modulus check'] + * found a coefficient outside [0,q-1]. Returned by check_pk and by the + * encapsulation API. */ +#define MLK_ERR_INVALID_PK (-4) +/* Secret key validation failed: the @[FIPS203, Section 7.3, 'hash check'] + * found the embedded public key hash inconsistent. Returned by check_sk and + * by the decapsulation API. */ +#define MLK_ERR_INVALID_SK (-5) +/* The 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and + * @[FIPS203, Section 7.1, Pairwise Consistency] failed. Only possible when + * MLK_CONFIG_KEYGEN_PCT is enabled; signals that the freshly generated key + * pair failed its encaps/decaps self-test. */ +#define MLK_ERR_PCT_FAIL (-6) + +/********************* Namespacing and Qualifiers *****************************/ + +#define MLK_API_CONCAT_(x, y) x##y +#define MLK_API_CONCAT(x, y) MLK_API_CONCAT_(x, y) +#define MLK_API_CONCAT_UNDERSCORE(x, y) MLK_API_CONCAT(MLK_API_CONCAT(x, _), y) + +/* You need to make sure the config file is in the include path. */ +#if defined(MLK_CONFIG_FILE) +#include MLK_CONFIG_FILE +#else +#include "mlkem_native_config.h" +#endif + +/* Namespace prefix for the public API symbols. For multi-level builds, the + * parameter set is appended to disambiguate the security levels. */ +#if defined(MLK_CONFIG_MULTILEVEL_BUILD) +#define MLK_API_NAMESPACE_PREFIX \ + MLK_API_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, MLK_CONFIG_PARAMETER_SET) +#else +#define MLK_API_NAMESPACE_PREFIX MLK_CONFIG_NAMESPACE_PREFIX +#endif + +#define MLK_API_NAMESPACE(sym) \ + MLK_API_CONCAT_UNDERSCORE(MLK_API_NAMESPACE_PREFIX, sym) + +#if defined(__GNUC__) || defined(__clang__) +#define MLK_API_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result)) +#else +#define MLK_API_MUST_CHECK_RETURN_VALUE +#endif + +#if defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER) +#define MLK_API_QUALIFIER MLK_CONFIG_EXTERNAL_API_QUALIFIER +#else +#define MLK_API_QUALIFIER +#endif + +/****************************** Function API **********************************/ + +#if !defined(MLK_CONFIG_CONSTANTS_ONLY) + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].} + * + * @param[out] pk Output public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[out] sk Output private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param[in] coins Input randomness, an array of 2*MLKEM_SYMBYTES uniformly + * random bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL MLK_CONFIG_KEYGEN_PCT enabled and random + * number generation failed within the PCT. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(keypair_derand)( + uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + const uint8_t coins[2 * MLKEM_SYMBYTES] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].} + * + * @param[out] pk Output public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[out] sk Output private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(keypair)( + uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].} + * + * @param[out] ct Output ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param[in] coins Input randomness, an array of MLKEM_SYMBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(enc_derand)( + uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t ss[MLKEM_BYTES], + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)], + const uint8_t coins[MLKEM_SYMBYTES] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].} + * + * @param[out] ct Output ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(enc)( + uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET)], + uint8_t ss[MLKEM_BYTES], + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/** + * Generate shared secret for a given ciphertext and private key. + * + * @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].} + * + * @param[out] ss Output shared secret, an array of MLKEM_BYTES bytes. + * @param[in] ct Input ciphertext, an array of + * MLKEM{512,768,1024}_CIPHERTEXTBYTES bytes. + * @param[in] sk Input private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK The 'hash check' @[FIPS203, Section 7.3] + * for the secret key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(dec)( + uint8_t ss[MLKEM_BYTES], + const uint8_t ct[MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET)], + const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + + +/** + * Implements modulus check mandated by FIPS 203, i.e., ensures that + * coefficients are in [0,q-1]. + * + * @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].} + * + * @param[in] pk Input public key, an array of + * MLKEM{512,768,1024}_PUBLICKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK Modulus check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(check_pk)( + const uint8_t pk[MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +/** + * Implements public key hash check mandated by FIPS 203, i.e., ensures that + * sk[768𝑘+32 ∶ 768𝑘+64] = H(pk) = H(sk[384𝑘 : 768𝑘+32]). + * + * @spec{Implements @[FIPS203, Section 7.3, 'hash check'].} + * + * @param[in] sk Input private key, an array of + * MLKEM{512,768,1024}_SECRETKEYBYTES bytes. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK Public key hash check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_API_QUALIFIER +MLK_API_MUST_CHECK_RETURN_VALUE +int MLK_API_NAMESPACE(check_sk)( + const uint8_t sk[MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET)] +#ifdef MLK_CONFIG_CONTEXT_PARAMETER + , + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context +#endif +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#ifdef __cplusplus +} +#endif + +#undef MLK_API_NAMESPACE_PREFIX + +#endif /* !MLK_CONFIG_CONSTANTS_ONLY */ + + +/***************************** Memory Usage **********************************/ + +/* + * By default mlkem-native performs all memory allocations on the stack. + * Alternatively, mlkem-native supports custom allocation of large structures + * through the `MLK_CONFIG_CUSTOM_ALLOC_FREE` configuration option. + * See mlkem_native_config.h for details. + * + * `MLK_TOTAL_ALLOC_{512,768,1024}_{KEYPAIR,ENCAPS,DECAPS}` indicates the + * maximum (accumulative) allocation via MLK_ALLOC for each parameter set and + * operation. Note that some stack allocation remains even when using custom + * allocators, so these values are lower than total stack usage with the default + * stack-only allocation. + * + * These constants may be used to implement custom allocations using a + * fixed-sized buffer and a simple allocator (e.g., bump allocator). + */ +/* check-magic: off */ +#define MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT 5824 +#define MLK_TOTAL_ALLOC_512_KEYPAIR_PCT 10048 +#define MLK_TOTAL_ALLOC_512_ENCAPS 8384 +#define MLK_TOTAL_ALLOC_512_DECAPS 9152 +#define MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT 10176 +#define MLK_TOTAL_ALLOC_768_KEYPAIR_PCT 15552 +#define MLK_TOTAL_ALLOC_768_ENCAPS 13248 +#define MLK_TOTAL_ALLOC_768_DECAPS 14336 +#define MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT 15552 +#define MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT 22400 +#define MLK_TOTAL_ALLOC_1024_ENCAPS 19136 +#define MLK_TOTAL_ALLOC_1024_DECAPS 20704 +/* check-magic: on */ + +/* + * MLK_TOTAL_ALLOC_*_KEYPAIR adapts based on MLK_CONFIG_KEYGEN_PCT. + */ +#if defined(MLK_CONFIG_KEYGEN_PCT) +#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#else +#define MLK_TOTAL_ALLOC_512_KEYPAIR MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#define MLK_TOTAL_ALLOC_768_KEYPAIR MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#define MLK_TOTAL_ALLOC_1024_KEYPAIR MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#endif + +#define MLK_MAX3_(a, b, c) \ + ((a) > (b) ? ((a) > (c) ? (a) : (c)) : ((b) > (c) ? (b) : (c))) + +/* + * `MLK_TOTAL_ALLOC_{512,768,1024}` is the maximum across all operations for + * each parameter set. + */ +#define MLK_TOTAL_ALLOC_512 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_512_KEYPAIR, MLK_TOTAL_ALLOC_512_ENCAPS, \ + MLK_TOTAL_ALLOC_512_DECAPS) +#define MLK_TOTAL_ALLOC_768 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_768_KEYPAIR, MLK_TOTAL_ALLOC_768_ENCAPS, \ + MLK_TOTAL_ALLOC_768_DECAPS) +#define MLK_TOTAL_ALLOC_1024 \ + MLK_MAX3_(MLK_TOTAL_ALLOC_1024_KEYPAIR, MLK_TOTAL_ALLOC_1024_ENCAPS, \ + MLK_TOTAL_ALLOC_1024_DECAPS) + +#endif /* !MLK_H */ diff --git a/src/mlkem-native/mlkem/mlkem_native_asm.S b/src/mlkem-native/mlkem/mlkem_native_asm.S new file mode 100644 index 0000000000..a2f1cf192a --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native_asm.S @@ -0,0 +1,716 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +/****************************************************************************** + * + * Single assembly unit for fixed-level build of mlkem-native + * + * This assembly unit bundles together all assembly files for a build + * of mlkem-native for a fixed security level (MLKEM-512/768/1024). + * + * # Multi-level build + * + * If you want an SCU build of mlkem-native with support for multiple security + * levels, you should include this file once with + * MLK_CONFIG_MULTILEVEL_WITH_SHARED set. + * + * (You could also follow the same pattern as for mlkem_native.c + * and include it for every level, setting MLK_CONFIG_MULTILEVEL_NO_SHARED + * for all but one. For builds with MLK_CONFIG_MULTILEVEL_NO_SHARED, this + * file will then be ignored.) + * + * # Configuration + * + * The following options from the mlkem-native configuration are relevant: + * + * - MLK_CONFIG_FIPS202_CUSTOM_HEADER + * Set this option if you use a custom FIPS202 implementation. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * Set this option if you want to include the native arithmetic backends + * in your build. + * + * - MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * Set this option if you want to include the native FIPS202 backends + * in your build. + * + * - MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * Set this option if you want to keep the directives defined in + * level-independent headers. This is needed for a multi-level build. + */ + +/* If parts of the mlkem-native source tree are not used, + * consider reducing this header via `unifdef`. + * + * Example: + * ```bash + * unifdef -UMLK_CONFIG_USE_NATIVE_BACKEND_ARITH mlkem_native_asm.S + * ``` + */ + +#include "src/common.h" + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#if defined(MLK_SYS_AARCH64) +#include "src/native/aarch64/src/mlkem_intt_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_ntt_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S" +#include "src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S" +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +#include "src/native/x86_64/src/mlkem_intt_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_ntt_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_reduce_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S" +#include "src/native/x86_64/src/mlkem_tomont_avx2_asm.S" +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +#endif +#if defined(MLK_SYS_PPC64LE) +#include "src/native/ppc64le/src/mlkem_intt_ppc_asm.S" +#include "src/native/ppc64le/src/mlkem_ntt_ppc_asm.S" +#include "src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S" +#include "src/native/ppc64le/src/mlkem_reduce_ppc_asm.S" +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#if defined(MLK_SYS_AARCH64) +#include "src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S" +#include "src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S" +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +#include "src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S" +#endif +#if defined(MLK_SYS_ARMV81M_MVE) +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S" +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S" +#include "src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + + +/* Macro #undef's + * + * The following undefines macros from headers + * included by the source files imported above. + * + * This is to allow building and linking multiple builds + * of mlkem-native for varying parameter sets through concatenation + * of this file, as if the files had been compiled separately. + * If this is not relevant to you, you may remove the following. + * + * NOTE: This is not needed for the assembly SCU since, at present, + * there is no need to include it multiple times. + * We keep it for uniformity with mlkem_native.c only. + * + * NOTE: To avoid having to distinguish between which headers are included + * from the assembly files, we #undef the same set of directives + * as in mlkem_native.c + */ + +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-specific files + */ +/* mlkem/mlkem_native.h */ +#undef MLKEM1024_BYTES +#undef MLKEM1024_CIPHERTEXTBYTES +#undef MLKEM1024_PUBLICKEYBYTES +#undef MLKEM1024_SECRETKEYBYTES +#undef MLKEM1024_SYMBYTES +#undef MLKEM512_BYTES +#undef MLKEM512_CIPHERTEXTBYTES +#undef MLKEM512_PUBLICKEYBYTES +#undef MLKEM512_SECRETKEYBYTES +#undef MLKEM512_SYMBYTES +#undef MLKEM768_BYTES +#undef MLKEM768_CIPHERTEXTBYTES +#undef MLKEM768_PUBLICKEYBYTES +#undef MLKEM768_SECRETKEYBYTES +#undef MLKEM768_SYMBYTES +#undef MLKEM_BYTES +#undef MLKEM_CIPHERTEXTBYTES +#undef MLKEM_CIPHERTEXTBYTES_ +#undef MLKEM_PUBLICKEYBYTES +#undef MLKEM_PUBLICKEYBYTES_ +#undef MLKEM_SECRETKEYBYTES +#undef MLKEM_SECRETKEYBYTES_ +#undef MLKEM_SYMBYTES +#undef MLK_API_CONCAT +#undef MLK_API_CONCAT_ +#undef MLK_API_CONCAT_UNDERSCORE +#undef MLK_API_MUST_CHECK_RETURN_VALUE +#undef MLK_API_NAMESPACE +#undef MLK_API_NAMESPACE_PREFIX +#undef MLK_API_QUALIFIER +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_H +#undef MLK_MAX3_ +#undef MLK_TOTAL_ALLOC_1024 +#undef MLK_TOTAL_ALLOC_1024_DECAPS +#undef MLK_TOTAL_ALLOC_1024_ENCAPS +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_1024_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_512 +#undef MLK_TOTAL_ALLOC_512_DECAPS +#undef MLK_TOTAL_ALLOC_512_ENCAPS +#undef MLK_TOTAL_ALLOC_512_KEYPAIR +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_512_KEYPAIR_PCT +#undef MLK_TOTAL_ALLOC_768 +#undef MLK_TOTAL_ALLOC_768_DECAPS +#undef MLK_TOTAL_ALLOC_768_ENCAPS +#undef MLK_TOTAL_ALLOC_768_KEYPAIR +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_NO_PCT +#undef MLK_TOTAL_ALLOC_768_KEYPAIR_PCT +/* mlkem/src/common.h */ +#undef MLK_ADD_PARAM_SET +#undef MLK_ALLOC +#undef MLK_APPLY +#undef MLK_ASM_FN_SIZE +#undef MLK_ASM_FN_SYMBOL +#undef MLK_ASM_NAMESPACE +#undef MLK_BUILD_INTERNAL +#undef MLK_COMMON_H +#undef MLK_CONCAT +#undef MLK_CONCAT_ +#undef MLK_EMPTY_CU +#undef MLK_ERR_FAIL +#undef MLK_ERR_INVALID_PK +#undef MLK_ERR_INVALID_SK +#undef MLK_ERR_OUT_OF_MEMORY +#undef MLK_ERR_PCT_FAIL +#undef MLK_ERR_RNG_FAIL +#undef MLK_EXTERNAL_API +#undef MLK_FIPS202X4_HEADER_FILE +#undef MLK_FIPS202_HEADER_FILE +#undef MLK_FREE +#undef MLK_INTERNAL_API +#undef MLK_INTERNAL_DATA_DECLARATION +#undef MLK_INTERNAL_DATA_DEFINITION +#undef MLK_NAMESPACE +#undef MLK_NAMESPACE_K +#undef MLK_NAMESPACE_PREFIX +#undef MLK_NAMESPACE_PREFIX_K +#undef mlk_memcpy +#undef mlk_memset +/* mlkem/src/indcpa.h */ +#undef MLK_INDCPA_H +#undef mlk_gen_matrix +#undef mlk_indcpa_dec +#undef mlk_indcpa_enc +#undef mlk_indcpa_keypair_derand +/* mlkem/src/kem.h */ +#undef MLK_KEM_H +#undef mlk_kem_check_pk +#undef mlk_kem_check_sk +#undef mlk_kem_dec +#undef mlk_kem_enc +#undef mlk_kem_enc_derand +#undef mlk_kem_keypair +#undef mlk_kem_keypair_derand +/* mlkem/src/params.h */ +#undef MLKEM_DU +#undef MLKEM_DV +#undef MLKEM_ETA1 +#undef MLKEM_ETA2 +#undef MLKEM_INDCCA_CIPHERTEXTBYTES +#undef MLKEM_INDCCA_PUBLICKEYBYTES +#undef MLKEM_INDCCA_SECRETKEYBYTES +#undef MLKEM_INDCPA_BYTES +#undef MLKEM_INDCPA_MSGBYTES +#undef MLKEM_INDCPA_PUBLICKEYBYTES +#undef MLKEM_INDCPA_SECRETKEYBYTES +#undef MLKEM_K +#undef MLKEM_N +#undef MLKEM_POLYBYTES +#undef MLKEM_POLYCOMPRESSEDBYTES_D10 +#undef MLKEM_POLYCOMPRESSEDBYTES_D11 +#undef MLKEM_POLYCOMPRESSEDBYTES_D4 +#undef MLKEM_POLYCOMPRESSEDBYTES_D5 +#undef MLKEM_POLYCOMPRESSEDBYTES_DU +#undef MLKEM_POLYCOMPRESSEDBYTES_DV +#undef MLKEM_POLYVECBYTES +#undef MLKEM_POLYVECCOMPRESSEDBYTES_DU +#undef MLKEM_Q +#undef MLKEM_Q_HALF +#undef MLKEM_SSBYTES +#undef MLKEM_SYMBYTES +#undef MLKEM_UINT12_LIMIT +#undef MLK_PARAMS_H +/* mlkem/src/poly_k.h */ +#undef MLK_POLY_K_H +#undef mlk_poly_compress_du +#undef mlk_poly_compress_dv +#undef mlk_poly_decompress_du +#undef mlk_poly_decompress_dv +#undef mlk_poly_getnoise_eta1122_4x +#undef mlk_poly_getnoise_eta1_4x +#undef mlk_poly_getnoise_eta2 +#undef mlk_poly_getnoise_eta2_4x +#undef mlk_polymat +#undef mlk_polyvec +#undef mlk_polyvec_add +#undef mlk_polyvec_basemul_acc_montgomery_cached +#undef mlk_polyvec_compress_du +#undef mlk_polyvec_decompress_du +#undef mlk_polyvec_frombytes +#undef mlk_polyvec_invntt_tomont +#undef mlk_polyvec_mulcache +#undef mlk_polyvec_mulcache_compute +#undef mlk_polyvec_ntt +#undef mlk_polyvec_reduce +#undef mlk_polyvec_tobytes +#undef mlk_polyvec_tomont + +#if !defined(MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS) +/* + * Undefine macros from MLK_CONFIG_PARAMETER_SET-generic files + */ +/* mlkem/src/compress.h */ +#undef MLK_COMPRESS_H +#undef mlk_poly_compress_d10 +#undef mlk_poly_compress_d11 +#undef mlk_poly_compress_d4 +#undef mlk_poly_compress_d5 +#undef mlk_poly_decompress_d10 +#undef mlk_poly_decompress_d11 +#undef mlk_poly_decompress_d4 +#undef mlk_poly_decompress_d5 +#undef mlk_poly_frombytes +#undef mlk_poly_frommsg +#undef mlk_poly_tobytes +#undef mlk_poly_tomsg +/* mlkem/src/context.h */ +#undef MLK_CONTEXT_H +#undef MLK_CONTEXT_PARAMETERS_0 +#undef MLK_CONTEXT_PARAMETERS_1 +#undef MLK_CONTEXT_PARAMETERS_2 +#undef MLK_CONTEXT_PARAMETERS_3 +#undef MLK_CONTEXT_PARAMETERS_4 +#undef MLK_CONTEXT_UNUSED +/* mlkem/src/debug.h */ +#undef MLK_DEBUG_H +#undef mlk_assert +#undef mlk_assert_abs_bound +#undef mlk_assert_abs_bound_2d +#undef mlk_assert_bound +#undef mlk_assert_bound_2d +#undef mlk_debug_check_assert +#undef mlk_debug_check_bounds +/* mlkem/src/poly.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NTT_BOUND +#undef MLK_POLY_H +#undef mlk_poly_add +#undef mlk_poly_invntt_tomont +#undef mlk_poly_mulcache_compute +#undef mlk_poly_ntt +#undef mlk_poly_reduce +#undef mlk_poly_sub +#undef mlk_poly_tomont +/* mlkem/src/randombytes.h */ +#undef MLK_RANDOMBYTES_H +/* mlkem/src/sampling.h */ +#undef MLK_SAMPLING_H +#undef mlk_poly_cbd2 +#undef mlk_poly_cbd3 +#undef mlk_poly_rej_uniform +#undef mlk_poly_rej_uniform_x4 +/* mlkem/src/symmetric.h */ +#undef MLK_SYMMETRIC_H +#undef MLK_XOF_RATE +#undef mlk_hash_g +#undef mlk_hash_h +#undef mlk_hash_j +#undef mlk_prf_eta +#undef mlk_prf_eta1 +#undef mlk_prf_eta1_x4 +#undef mlk_prf_eta2 +#undef mlk_xof_absorb +#undef mlk_xof_ctx +#undef mlk_xof_init +#undef mlk_xof_release +#undef mlk_xof_squeezeblocks +#undef mlk_xof_x4_absorb +#undef mlk_xof_x4_ctx +#undef mlk_xof_x4_init +#undef mlk_xof_x4_release +#undef mlk_xof_x4_squeezeblocks +/* mlkem/src/sys.h */ +#undef MLK_ALIGN +#undef MLK_ALIGN_UP +#undef MLK_ALWAYS_INLINE +#undef MLK_CET_ENDBR +#undef MLK_CT_TESTING_DECLASSIFY +#undef MLK_CT_TESTING_SECRET +#undef MLK_DEFAULT_ALIGN +#undef MLK_HAVE_INLINE_ASM +#undef MLK_INLINE +#undef MLK_MUST_CHECK_RETURN_VALUE +#undef MLK_NOINLINE +#undef MLK_RESTRICT +#undef MLK_STATIC_TESTABLE +#undef MLK_SYSV_ABI +#undef MLK_SYSV_ABI_SUPPORTED +#undef MLK_SYS_AARCH64 +#undef MLK_SYS_AARCH64_EB +#undef MLK_SYS_AARCH64_NEON +#undef MLK_SYS_APPLE +#undef MLK_SYS_ARMV81M_MVE +#undef MLK_SYS_BIG_ENDIAN +#undef MLK_SYS_H +#undef MLK_SYS_LINUX +#undef MLK_SYS_LITTLE_ENDIAN +#undef MLK_SYS_PPC64LE +#undef MLK_SYS_RISCV32 +#undef MLK_SYS_RISCV64 +#undef MLK_SYS_RISCV64_RVV +#undef MLK_SYS_WINDOWS +#undef MLK_SYS_X86_64 +#undef MLK_SYS_X86_64_AVX2 +/* mlkem/src/verify.h */ +#undef MLK_USE_ASM_VALUE_BARRIER +#undef MLK_VERIFY_H +#undef mlk_ct_opt_blocker_u64 +/* mlkem/src/cbmc.h */ +#undef MLK_CBMC_H +#undef __contract__ +#undef __loop__ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +/* + * Undefine macros from FIPS-202 files + */ +/* mlkem/src/fips202/fips202.h */ +#undef FIPS202_X4_DEFAULT_IMPLEMENTATION +#undef MLK_FIPS202_FIPS202_H +#undef SHA3_256_HASHBYTES +#undef SHA3_256_RATE +#undef SHA3_384_RATE +#undef SHA3_512_HASHBYTES +#undef SHA3_512_RATE +#undef SHAKE128_RATE +#undef SHAKE256_RATE +#undef mlk_sha3_256 +#undef mlk_sha3_512 +#undef mlk_shake128_absorb_once +#undef mlk_shake128_init +#undef mlk_shake128_release +#undef mlk_shake128_squeezeblocks +#undef mlk_shake256 +/* mlkem/src/fips202/fips202x4.h */ +#undef MLK_FIPS202_FIPS202X4_H +#undef mlk_shake128x4_absorb_once +#undef mlk_shake128x4_init +#undef mlk_shake128x4_release +#undef mlk_shake128x4_squeezeblocks +#undef mlk_shake256x4 +/* mlkem/src/fips202/keccakf1600.h */ +#undef MLK_FIPS202_KECCAKF1600_H +#undef MLK_KECCAK_LANES +#undef MLK_KECCAK_WAY +#undef mlk_keccakf1600_extract_bytes +#undef mlk_keccakf1600_permute +#undef mlk_keccakf1600_xor_bytes +#undef mlk_keccakf1600x4_extract_bytes +#undef mlk_keccakf1600x4_permute +#undef mlk_keccakf1600x4_xor_bytes +#endif /* !MLK_CONFIG_FIPS202_CUSTOM_HEADER */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* mlkem/src/fips202/native/api.h */ +#undef MLK_FIPS202_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +/* mlkem/src/fips202/native/auto.h */ +#undef MLK_FIPS202_NATIVE_AUTO_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (FIPS202, AArch64) + */ +/* mlkem/src/fips202/native/aarch64/auto.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h */ +#undef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#undef mlk_keccak_f1600_x1_scalar_aarch64_asm +#undef mlk_keccak_f1600_x1_v84a_aarch64_asm +#undef mlk_keccak_f1600_x2_v84a_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm +#undef mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm +#undef mlk_keccakf1600_round_constants +/* mlkem/src/fips202/native/aarch64/x1_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#undef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x1_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X1_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X1 +/* mlkem/src/fips202/native/aarch64/x2_v84a.h */ +#undef MLK_FIPS202_AARCH64_NEED_X2_V84A +#undef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h */ +#undef MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID +#undef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (FIPS202, x86_64) + */ +/* mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h */ +#undef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#undef MLK_FIPS202_X86_64_NEED_X4_AVX2 +#undef MLK_USE_NATIVE_FIPS202_X4 +/* mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h */ +#undef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#undef mlk_keccak_f1600_x4_avx2_asm +#undef mlk_keccak_rho56 +#undef mlk_keccak_rho8 +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_ARMV81M_MVE) +/* + * Undefine macros from native code (FIPS202, Armv8.1-M) + */ +/* mlkem/src/fips202/native/armv81m/mve.h */ +#undef MLK_FIPS202_ARMV81M_NEED_X4 +#undef MLK_FIPS202_NATIVE_ARMV81M +#undef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#undef MLK_USE_NATIVE_FIPS202_X4 +#undef MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES +#undef MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES +#undef mlk_keccak_f1600_x4_native_impl +/* mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h */ +#undef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#undef mlk_keccak_f1600_x4_mve_asm +#undef mlk_keccak_f1600_x4_state_extract_bytes_asm +#undef mlk_keccak_f1600_x4_state_xor_bytes_asm +#undef mlk_keccakf1600_round_constants +#endif /* MLK_SYS_ARMV81M_MVE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* mlkem/src/native/api.h */ +#undef MLK_INVNTT_BOUND +#undef MLK_NATIVE_API_H +#undef MLK_NATIVE_FUNC_FALLBACK +#undef MLK_NATIVE_FUNC_SUCCESS +#undef MLK_NTT_BOUND +/* mlkem/src/native/meta.h */ +#undef MLK_NATIVE_META_H +#if defined(MLK_SYS_AARCH64) +/* + * Undefine macros from native code (Arith, AArch64) + */ +/* mlkem/src/native/aarch64/meta.h */ +#undef MLK_ARITH_BACKEND_AARCH64 +#undef MLK_NATIVE_AARCH64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/aarch64/src/arith_native_aarch64.h */ +#undef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#undef mlk_aarch64_invntt_zetas_layer12345 +#undef mlk_aarch64_invntt_zetas_layer67 +#undef mlk_aarch64_ntt_zetas_layer12345 +#undef mlk_aarch64_ntt_zetas_layer67 +#undef mlk_aarch64_zetas_mulcache_native +#undef mlk_aarch64_zetas_mulcache_twisted_native +#undef mlk_intt_aarch64_asm +#undef mlk_ntt_aarch64_asm +#undef mlk_poly_mulcache_compute_aarch64_asm +#undef mlk_poly_reduce_aarch64_asm +#undef mlk_poly_tobytes_aarch64_asm +#undef mlk_poly_tomont_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm +#undef mlk_rej_uniform_aarch64_asm +#undef mlk_rej_uniform_table +#endif /* MLK_SYS_AARCH64 */ +#if defined(MLK_SYS_X86_64) +/* + * Undefine macros from native code (Arith, X86_64) + */ +/* mlkem/src/native/x86_64/meta.h */ +#undef MLK_ARITH_BACKEND_X86_64_DEFAULT +#undef MLK_NATIVE_X86_64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_COMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_COMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D11 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#undef MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#undef MLK_USE_NATIVE_POLY_FROMBYTES +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOBYTES +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/x86_64/src/arith_native_x86_64.h */ +#undef MLK_AVX2_REJ_UNIFORM_BUFLEN +#undef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#undef mlk_invntt_avx2_asm +#undef mlk_ntt_avx2_asm +#undef mlk_nttfrombytes_avx2_asm +#undef mlk_ntttobytes_avx2_asm +#undef mlk_nttunpack_avx2_asm +#undef mlk_poly_compress_d10_avx2_asm +#undef mlk_poly_compress_d11_avx2_asm +#undef mlk_poly_compress_d4_avx2_asm +#undef mlk_poly_compress_d5_avx2_asm +#undef mlk_poly_decompress_d10_avx2_asm +#undef mlk_poly_decompress_d11_avx2_asm +#undef mlk_poly_decompress_d4_avx2_asm +#undef mlk_poly_decompress_d5_avx2_asm +#undef mlk_poly_mulcache_compute_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm +#undef mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm +#undef mlk_reduce_avx2_asm +#undef mlk_rej_uniform_avx2_asm +#undef mlk_rej_uniform_table +#undef mlk_tomont_avx2_asm +/* mlkem/src/native/x86_64/src/compress_consts.h */ +#undef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#undef mlk_compress_d10_data +#undef mlk_compress_d11_data +#undef mlk_compress_d4_data +#undef mlk_compress_d5_data +#undef mlk_decompress_d10_data +#undef mlk_decompress_d11_data +#undef mlk_decompress_d4_data +#undef mlk_decompress_d5_data +/* mlkem/src/native/x86_64/src/consts.h */ +#undef MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB +#undef MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD +#undef MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP +#undef MLK_NATIVE_X86_64_SRC_CONSTS_H +#undef mlk_qdata +#endif /* MLK_SYS_X86_64 */ +#if defined(MLK_SYS_RISCV64) +/* + * Undefine macros from native code (Arith, RISC-V 64) + */ +/* mlkem/src/native/riscv64/meta.h */ +#undef MLK_ARITH_BACKEND_RISCV64 +#undef MLK_NATIVE_RISCV64_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#undef MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +#undef MLK_USE_NATIVE_REJ_UNIFORM +/* mlkem/src/native/riscv64/src/arith_native_riscv64.h */ +#undef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#undef mlk_rv64v_poly_add +#undef mlk_rv64v_poly_basemul_mont_add_k2 +#undef mlk_rv64v_poly_basemul_mont_add_k3 +#undef mlk_rv64v_poly_basemul_mont_add_k4 +#undef mlk_rv64v_poly_invntt_tomont +#undef mlk_rv64v_poly_ntt +#undef mlk_rv64v_poly_reduce +#undef mlk_rv64v_poly_sub +#undef mlk_rv64v_poly_tomont +#undef mlk_rv64v_rej_uniform +/* mlkem/src/native/riscv64/src/rv64v_debug.h */ +#undef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#undef mlk_assert_abs_bound_int16m1 +#undef mlk_assert_abs_bound_int16m2 +#undef mlk_assert_bound_int16m1 +#undef mlk_assert_bound_int16m2 +#undef mlk_debug_check_bounds_int16m1 +#undef mlk_debug_check_bounds_int16m2 +#endif /* MLK_SYS_RISCV64 */ +#if defined(MLK_SYS_PPC64LE) +/* + * Undefine macros from native code (Arith, PPC64LE) + */ +/* mlkem/src/native/ppc64le/meta.h */ +#undef MLK_ARITH_BACKEND_NAME +#undef MLK_ARITH_BACKEND_PPC64LE_DEFAULT +#undef MLK_NATIVE_PPC64LE_META_H +#undef MLK_USE_NATIVE_INTT +#undef MLK_USE_NATIVE_NTT +#undef MLK_USE_NATIVE_POLY_REDUCE +#undef MLK_USE_NATIVE_POLY_TOMONT +/* mlkem/src/native/ppc64le/src/arith_native_ppc64le.h */ +#undef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#undef mlk_intt_ppc_asm +#undef mlk_ntt_ppc_asm +#undef mlk_poly_tomont_ppc_asm +#undef mlk_reduce_ppc_asm +/* mlkem/src/native/ppc64le/src/consts.h */ +#undef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#undef MLK_PPC_C20159_OFFSET +#undef MLK_PPC_NQ_OFFSET +#undef MLK_PPC_N_INV_OFFSET +#undef MLK_PPC_N_INV_TW_OFFSET +#undef MLK_PPC_Q_OFFSET +#undef MLK_PPC_TOMONT_OFFSET +#undef MLK_PPC_TOMONT_TW_OFFSET +#undef MLK_PPC_ZETA_INTT_OFFSET +#undef MLK_PPC_ZETA_INTT_TW_OFFSET +#undef MLK_PPC_ZETA_NTT_OFFSET +#undef MLK_PPC_ZETA_NTT_TW_OFFSET +#undef mlk_ppc_qdata +#endif /* MLK_SYS_PPC64LE */ +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif /* !MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ diff --git a/src/mlkem-native/mlkem/mlkem_native_config.h b/src/mlkem-native/mlkem/mlkem_native_config.h new file mode 100644 index 0000000000..a4b204b68e --- /dev/null +++ b/src/mlkem-native/mlkem/mlkem_native_config.h @@ -0,0 +1,683 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_CONFIG_H +#define MLK_CONFIG_H + +/** + * Specifies the parameter set for ML-KEM: + * - MLK_CONFIG_PARAMETER_SET=512 corresponds to ML-KEM-512 + * - MLK_CONFIG_PARAMETER_SET=768 corresponds to ML-KEM-768 + * - MLK_CONFIG_PARAMETER_SET=1024 corresponds to ML-KEM-1024 + * + * If you want to support multiple parameter sets, build the library multiple + * times and set MLK_CONFIG_MULTILEVEL_BUILD. See MLK_CONFIG_MULTILEVEL_BUILD + * for how to do this while minimizing code duplication. + * + * This can also be set using CFLAGS. + */ +#ifndef MLK_CONFIG_PARAMETER_SET +#define MLK_CONFIG_PARAMETER_SET \ + 768 /* Change this for different security strengths */ +#endif + +/** + * MLK_CONFIG_FILE + * + * If defined, this is a header that will be included instead of the default + * configuration file mlkem/mlkem_native_config.h. + * + * When you need to build mlkem-native in multiple configurations, using + * varying MLK_CONFIG_FILE can be more convenient than configuring everything + * through CFLAGS. + * + * To use, MLK_CONFIG_FILE _must_ be defined prior to the inclusion of any + * mlkem-native headers. For example, it can be set by passing + * `-DMLK_CONFIG_FILE="..."` on the command line. + */ +/* #define MLK_CONFIG_FILE "mlkem_native_config.h" */ + +/** + * The prefix to use to namespace global symbols from mlkem/. + * + * In a multi-level build, level-dependent symbols will additionally be + * prefixed with the parameter set (512/768/1024). + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_NAMESPACE_PREFIX) +#define MLK_CONFIG_NAMESPACE_PREFIX MLK_DEFAULT_NAMESPACE_PREFIX +#endif + +/** + * MLK_CONFIG_MULTILEVEL_BUILD + * + * Set this if the build is part of a multi-level build supporting multiple + * parameter sets. + * + * If you need only a single parameter set, keep this unset. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_BUILD */ + +/** + * MLK_CONFIG_EXTERNAL_API_QUALIFIER + * + * If set, this option provides an additional function qualifier to be added + * to declarations of mlkem-native's public API. + * + * The primary use case for this option are single-CU builds where the public + * API exposed by mlkem-native is wrapped by another API in the consuming + * application. In this case, even mlkem-native's public API can be marked + * `static`. + */ +/* #define MLK_CONFIG_EXTERNAL_API_QUALIFIER */ + +/** + * MLK_CONFIG_NO_KEYPAIR_API + * + * By default, mlkem-native includes support for generating key pairs. + * If you don't need this, set MLK_CONFIG_NO_KEYPAIR_API to exclude + * keypair and keypair_derand, and all internal + * APIs only needed by those functions. + */ +/* #define MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * MLK_CONFIG_NO_ENCAPS_API + * + * By default, mlkem-native includes support for encapsulation. If you + * don't need this, set MLK_CONFIG_NO_ENCAPS_API to exclude + * enc, enc_derand, check_pk, and + * all internal APIs only needed by those functions. + * + * @note Setting this option is incompatible with MLK_CONFIG_KEYGEN_PCT + * as the current PCT implementation requires enc(). + */ +/* #define MLK_CONFIG_NO_ENCAPS_API */ + +/** + * MLK_CONFIG_NO_DECAPS_API + * + * By default, mlkem-native includes support for decapsulation. If you + * don't need this, set MLK_CONFIG_NO_DECAPS_API to exclude + * dec, check_sk, and all internal APIs only + * needed by those functions. + * + * @note Setting this option is incompatible with MLK_CONFIG_KEYGEN_PCT + * as the current PCT implementation requires dec(). + */ +/* #define MLK_CONFIG_NO_DECAPS_API */ + +/** + * MLK_CONFIG_NO_RANDOMIZED_API + * + * If this option is set, mlkem-native will be built without the randomized + * API functions (keypair and enc). This allows users + * to build mlkem-native without providing a randombytes() implementation + * if they only need the deterministic API (keypair_derand, + * enc_derand, dec). + * + * @note This option is incompatible with MLK_CONFIG_KEYGEN_PCT as the + * current PCT implementation requires enc(). + */ +/* #define MLK_CONFIG_NO_RANDOMIZED_API */ + +/** + * MLK_CONFIG_CONSTANTS_ONLY + * + * If you only need the size constants (MLKEM_PUBLICKEYBYTES, etc.) but no + * function declarations, set MLK_CONFIG_CONSTANTS_ONLY. + * + * This only affects the public header mlkem_native.h, not the + * implementation. + */ +/* #define MLK_CONFIG_CONSTANTS_ONLY */ + +/****************************************************************************** + * + * Build-only configuration options + * + * The remaining configurations are build-options only. + * They do not affect the API described in mlkem_native.h. + * + *****************************************************************************/ + +#if defined(MLK_BUILD_INTERNAL) +/** + * MLK_CONFIG_MULTILEVEL_WITH_SHARED + * + * This is for multi-level builds of mlkem-native only. If you need only a + * single parameter set, keep this unset. + * + * If this is set, all MLK_CONFIG_PARAMETER_SET-independent code will be + * included in the build, including code needed only for other parameter + * sets. + * + * Example: mlk_poly_cbd3 is only needed for MLK_CONFIG_PARAMETER_SET == 512. + * Yet, if this option is set for a build with + * MLK_CONFIG_PARAMETER_SET == 768/1024, it would be included. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_WITH_SHARED */ + +/** + * MLK_CONFIG_MULTILEVEL_NO_SHARED + * + * This is for multi-level builds of mlkem-native only. If you need only a + * single parameter set, keep this unset. + * + * If this is set, no MLK_CONFIG_PARAMETER_SET-independent code will be + * included in the build. + * + * To build mlkem-native with support for all parameter sets, build it three + * times -- once per parameter set -- and set the option + * MLK_CONFIG_MULTILEVEL_WITH_SHARED for exactly one of them, and + * MLK_CONFIG_MULTILEVEL_NO_SHARED for the others. + * MLK_CONFIG_MULTILEVEL_BUILD should be set for all of them. + * + * See examples/multilevel_build for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/** + * MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS + * + * This is only relevant for single compilation unit (SCU) builds of + * mlkem-native. In this case, it determines whether directives defined in + * parameter-set-independent headers should be #undef'ined or not at the + * end of the SCU file. This is needed in multilevel builds. + * + * See examples/multilevel_build_native for an example. + * + * This can also be set using CFLAGS. + */ +/* #define MLK_CONFIG_MONOBUILD_KEEP_SHARED_HEADERS */ + +/** + * MLK_CONFIG_USE_NATIVE_BACKEND_ARITH + * + * Determines whether a native arithmetic backend should be used. + * + * The arithmetic backend covers performance-critical functions such as the + * number-theoretic transform (NTT). + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the arithmetic backend to be used is determined + * by MLK_CONFIG_ARITH_BACKEND_FILE: if the latter is unset, the default + * backend for the target architecture will be used. If set, it must be the + * name of a backend metadata file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +/* #define MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ +#endif + +/** + * MLK_CONFIG_ARITH_BACKEND_FILE + * + * The arithmetic backend to use. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is unset, this option is ignored. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, this option must either + * be undefined or the filename of an arithmetic backend. If unset, the + * default backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLK_CONFIG_ARITH_BACKEND_FILE) +#define MLK_CONFIG_ARITH_BACKEND_FILE "native/meta.h" +#endif + +/** + * MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 + * + * Determines whether a native FIPS202 backend should be used. + * + * The FIPS202 backend covers 1x/2x/4x-fold Keccak-f1600, which is the + * performance bottleneck of SHA3 and SHAKE. + * + * If this option is unset, the C backend will be used. + * + * If this option is set, the FIPS202 backend to be used is determined by + * MLK_CONFIG_FIPS202_BACKEND_FILE: if the latter is unset, the default + * backend for the target architecture will be used. If set, it must be + * the name of a backend metadata file. + * + * This can also be set using CFLAGS. + */ +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +/* #define MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ +#endif + +/** + * MLK_CONFIG_FIPS202_BACKEND_FILE + * + * The FIPS-202 backend to use. + * + * If MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, this option must either + * be undefined or the filename of a FIPS202 backend. If unset, the default + * backend will be used. + * + * This can be set using CFLAGS. + */ +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLK_CONFIG_FIPS202_BACKEND_FILE) +#define MLK_CONFIG_FIPS202_BACKEND_FILE "fips202/native/auto.h" +#endif + +/** + * MLK_CONFIG_FIPS202_CUSTOM_HEADER + * + * Custom header to use for FIPS-202. + * + * This should only be set if you intend to use a custom FIPS-202 + * implementation, different from the one shipped with mlkem-native. + * + * If set, it must be the name of a file serving as the replacement for + * mlkem/src/fips202/fips202.h, and exposing the same API (see FIPS202.md). + */ +/* #define MLK_CONFIG_FIPS202_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLK_CONFIG_FIPS202X4_CUSTOM_HEADER + * + * Custom header to use for FIPS-202-X4. + * + * This should only be set if you intend to use a custom FIPS-202 + * implementation, different from the one shipped with mlkem-native. + * + * If set, it must be the name of a file serving as the replacement for + * mlkem/src/fips202/fips202x4.h, and exposing the same API (see FIPS202.md). + */ +/* #define MLK_CONFIG_FIPS202X4_CUSTOM_HEADER "SOME_FILE.h" */ + +/** + * MLK_CONFIG_CUSTOM_ZEROIZE + * + * In compliance with @[FIPS203, Section 3.3], mlkem-native zeroizes + * intermediate buffers before returning from function calls. By default, + * those buffers are allocated from the stack; if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is set, they are (mostly -- few exceptions remain at present) allocated from + * the configured custom allocator. + * + * mlkem-native also zeroizes caller-owned output buffers as needed to uphold + * the API convention that outputs be either unmodified or zeroized upon + * failure. + * + * Set this option and define `mlk_zeroize` if you want to use a custom + * method to zeroize intermediate and output buffers. + * + * The default implementation uses SecureZeroMemory on Windows and a + * memset + compiler barrier otherwise. If neither of those is available on + * the target platform, compilation will fail, and you will need to use + * MLK_CONFIG_CUSTOM_ZEROIZE to provide a custom implementation of + * `mlk_zeroize()`. + * + * @warning + * The zeroization conducted by mlkem-native reduces the likelihood of data + * leaking on the stack or custom allocators, but it does not eliminate it. + * For example, the C standard makes no guarantee about where a compiler + * allocates local structures and whether/where it makes copies of them. + * Also, in addition to entire structures, there may also be potentially + * exploitable leakage of individual values on the stack. If you need + * bullet-proof zeroization of the stack, you need to consider additional + * measures instead of what this feature provides. In this case, you can + * set mlk_zeroize to a no-op. Note that in this case you are also responsible + * for zeroizing output buffers upon failure. + */ +/* #define MLK_CONFIG_CUSTOM_ZEROIZE + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_RANDOMBYTES + * + * mlkem-native does not provide a secure randombytes implementation. Such + * an implementation has to be provided by the consumer. + * + * If this option is not set, mlkem-native expects a function + * int randombytes(uint8_t *out, size_t outlen). It is expected to return + * zero on success, and non-zero on failure. In case of failure, the + * top-level APIs will return an MLK_ERR_RNG_FAIL error code. + * + * Set this option and define `mlk_randombytes` (with the same signature + * and behaviour) if you want to use a custom method to sample randombytes + * with a different name or signature. + */ +/* #define MLK_CONFIG_CUSTOM_RANDOMBYTES + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE int mlk_randombytes(uint8_t *ptr, size_t len) + { + ... your implementation ... + return 0; + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + * + * mlkem-native backends may rely on specific hardware features. Those + * backends will only be included in an mlkem-native build if support for + * the respective features is enabled at compile-time. However, when + * building for a heterogeneous set of CPUs to run the resulting + * binary/library on, feature detection at _runtime_ is needed to decide + * whether a backend can be used or not. + * + * Set this option and define `mlk_sys_check_capability` if you want to + * use a custom method to dispatch between implementations. + * + * If this option is not set, mlkem-native uses compile-time feature + * detection only to decide which backend to use. + * + * If you compile mlkem-native on a system with different capabilities + * than the system that the resulting binary/library will be run on, you + * must use this option. + */ +/* #define MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) + __contract__( + ensures(return_value == 0 || return_value == 1) + ) + { + ... your implementation ... + } +*/ + +/** + * MLK_CONFIG_CUSTOM_ALLOC_FREE [EXPERIMENTAL] + * + * Set this option and define `MLK_CUSTOM_ALLOC` and `MLK_CUSTOM_FREE` if + * you want to use custom allocation for large local structures or buffers. + * + * By default, all buffers/structures are allocated on the stack. If this + * option is set, most of them will be allocated via MLK_CUSTOM_ALLOC. + * + * Parameters to MLK_CUSTOM_ALLOC: + * - T* v: Target pointer to declare. + * - T: Type of structure to be allocated. + * - N: Number of elements to be allocated. + * + * Parameters to MLK_CUSTOM_FREE: + * - T* v: Target pointer to free. May be NULL. + * - T: Type of structure to be freed. + * - N: Number of elements to be freed. + * + * @warning This option is experimental. Its scope, configuration and + * function/macro signatures may change at any time. We expect a + * stable API in a future version. + * + * @note Even if this option is set, some allocations further down the call + * stack will still be made from the stack, consuming up to 3KB of + * stack space. Those will likely be added to the scope of this + * option in the future. + * + * @note MLK_CUSTOM_ALLOC need not guarantee a successful allocation nor + * include error handling. Upon failure, the target pointer should + * simply be set to NULL. The calling code will handle this case and + * invoke MLK_CUSTOM_FREE. + */ +/* #define MLK_CONFIG_CUSTOM_ALLOC_FREE + #if !defined(__ASSEMBLER__) + #include + #define MLK_CUSTOM_ALLOC(v, T, N) \ + T* (v) = (T *)aligned_alloc(MLK_DEFAULT_ALIGN, \ + MLK_ALIGN_UP(sizeof(T) * (N))) + #define MLK_CUSTOM_FREE(v, T, N) free(v) + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_MEMCPY + * + * Set this option and define `mlk_memcpy` if you want to use a custom + * method to copy memory instead of the standard library memcpy function. + * + * The custom implementation must have the same signature and behavior as + * the standard memcpy function: + * void *mlk_memcpy(void *dest, const void *src, size_t n) + */ +/* #define MLK_CONFIG_CUSTOM_MEMCPY + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void *mlk_memcpy(void *dest, const void *src, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_CUSTOM_MEMSET + * + * Set this option and define `mlk_memset` if you want to use a custom + * method to set memory instead of the standard library memset function. + * + * The custom implementation must have the same signature and behavior as + * the standard memset function: + * void *mlk_memset(void *s, int c, size_t n) + */ +/* #define MLK_CONFIG_CUSTOM_MEMSET + #if !defined(__ASSEMBLER__) + #include + #include "src/sys.h" + static MLK_INLINE void *mlk_memset(void *s, int c, size_t n) + { + ... your implementation ... + } + #endif +*/ + +/** + * MLK_CONFIG_INTERNAL_API_QUALIFIER + * + * If set, this option provides an additional qualifier to be added to + * declarations of internal API functions and data. + * + * The primary use case for this option are single-CU builds, in which case + * this option can be set to `static`. + */ +/* #define MLK_CONFIG_INTERNAL_API_QUALIFIER */ + +/** + * MLK_CONFIG_CT_TESTING_ENABLED + * + * If set, mlkem-native annotates data as secret/public using valgrind's + * annotations VALGRIND_MAKE_MEM_UNDEFINED and VALGRIND_MAKE_MEM_DEFINED, + * enabling various checks for secret-dependent control flow or + * variable-time execution (depending on the exact version of valgrind + * installed). + */ +/* #define MLK_CONFIG_CT_TESTING_ENABLED */ + +/** + * MLK_CONFIG_NO_ASM + * + * If this option is set, mlkem-native will be built without use of native + * code or inline assembly. + * + * By default, inline assembly is used to implement value barriers. Without + * inline assembly, mlkem-native will use a global volatile 'opt blocker' + * instead; see verify.h. + * + * Inline assembly is also used to implement a secure zeroization function + * on non-Windows platforms. If this option is set and the target platform + * is not Windows, you MUST set MLK_CONFIG_CUSTOM_ZEROIZE and provide a + * custom zeroization function. + * + * If this option is set, MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 and + * MLK_CONFIG_USE_NATIVE_BACKEND_ARITH will be ignored, and no native + * backends will be used. + */ +/* #define MLK_CONFIG_NO_ASM */ + +/** + * MLK_CONFIG_NO_ASM_VALUE_BARRIER + * + * If this option is set, mlkem-native will be built without use of native + * code or inline assembly for value barriers. + * + * By default, inline assembly (if available) is used to implement value + * barriers. Without inline assembly, mlkem-native will use a global + * volatile 'opt blocker' instead; see verify.h. + */ +/* #define MLK_CONFIG_NO_ASM_VALUE_BARRIER */ + +/** + * MLK_CONFIG_KEYGEN_PCT + * + * Compliance with @[FIPS140_3_IG, p.87] requires a Pairwise Consistency + * Test (PCT) to be carried out on a freshly generated keypair before it + * can be exported. + * + * Set this option if such a check should be implemented. In this case, + * keypair_derand and keypair will return + * MLK_ERR_PCT_FAIL if the PCT failed. + * + * @note This feature will drastically lower the performance of key + * generation. + */ +/* #define MLK_CONFIG_KEYGEN_PCT */ + +/** + * MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + * + * If this option is set, the user must provide a runtime function + * `static inline int mlk_break_pct() { ... }` to indicate whether the PCT + * should be made to fail. + * + * This option only has an effect if MLK_CONFIG_KEYGEN_PCT is set. + */ +/* #define MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST + #if !defined(__ASSEMBLER__) + #include "src/sys.h" + static MLK_INLINE int mlk_break_pct(void) + { + ... return 0/1 depending on whether PCT should be broken ... + } + #endif +*/ + +/** + * MLK_CONFIG_SERIAL_FIPS202_ONLY + * + * Set this to use a FIPS202 implementation with global state that supports + * only one active Keccak computation at a time (e.g. some hardware + * accelerators). + * + * If this option is set, batched Keccak operations are disabled for + * rejection sampling during matrix generation. Instead, matrix entries + * will be generated one at a time. + * + * This allows offloading Keccak computations to a hardware accelerator + * that holds only a single Keccak state locally, rather than requiring + * support for batched (4x) Keccak states. + * + * @note Depending on the target CPU, disabling batched Keccak may reduce + * performance when using software FIPS202 implementations. Only + * enable this when you have to. + */ +/* #define MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +/** + * MLK_CONFIG_CONTEXT_PARAMETER + * + * Set this to add a context parameter that is provided to public API + * functions and is then available in custom callbacks. + * + * The type of the context parameter is configured via + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + */ +/* #define MLK_CONFIG_CONTEXT_PARAMETER */ + +/** + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE + * + * Set this to define the type for the context parameter used by + * MLK_CONFIG_CONTEXT_PARAMETER. + * + * This is only relevant if MLK_CONFIG_CONTEXT_PARAMETER is set. + */ +/* #define MLK_CONFIG_CONTEXT_PARAMETER_TYPE void* */ + +/************************* Config internals ********************************/ + +#endif /* MLK_BUILD_INTERNAL */ + +/* Default namespace + * + * Don't change this. If you need a different namespace, re-define + * MLK_CONFIG_NAMESPACE_PREFIX above instead, and remove the following. + * + * The default MLKEM namespace is + * + * PQCP_MLKEM_NATIVE_MLKEM_ + * + * e.g., PQCP_MLKEM_NATIVE_MLKEM512_ + */ + +#if defined(MLK_CONFIG_MULTILEVEL_BUILD) +/* In a multi-level build the parameter set is appended by the namespacing + * machinery, so the default prefix must not embed it. */ +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM +#elif MLK_CONFIG_PARAMETER_SET == 512 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM512 +#elif MLK_CONFIG_PARAMETER_SET == 768 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM768 +#elif MLK_CONFIG_PARAMETER_SET == 1024 +#define MLK_DEFAULT_NAMESPACE_PREFIX PQCP_MLKEM_NATIVE_MLKEM1024 +#endif + +#endif /* !MLK_CONFIG_H */ diff --git a/src/mlkem-native/mlkem/src/cbmc.h b/src/mlkem-native/mlkem/src/cbmc.h new file mode 100644 index 0000000000..cb24bb52a6 --- /dev/null +++ b/src/mlkem-native/mlkem/src/cbmc.h @@ -0,0 +1,222 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_CBMC_H +#define MLK_CBMC_H +/*************************************************** + * Basic replacements for __CPROVER_XXX contracts + ***************************************************/ +/* + * The `__contract__` / `__loop__` annotation macros use a + * leading-double-underscore spelling in line with other CBMC macros. + * clang-tidy flags these as reserved identifiers; we suppress the diagnostic + * at each definition site (NOLINT) rather than disabling the check globally, + * so it stays active for the rest of the tree. + */ +#ifndef CBMC + +/* clang-format off */ +#define __contract__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ + +#else /* !CBMC */ + + +/* clang-format off */ +#define __contract__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ + +/* https://diffblue.github.io/cbmc/contracts-assigns.html */ +#define assigns(...) __CPROVER_assigns(__VA_ARGS__) + +/* https://diffblue.github.io/cbmc/contracts-requires-ensures.html */ +#define requires(...) __CPROVER_requires(__VA_ARGS__) +#define ensures(...) __CPROVER_ensures(__VA_ARGS__) +/* https://diffblue.github.io/cbmc/contracts-loops.html */ +#define invariant(...) __CPROVER_loop_invariant(__VA_ARGS__) +#define decreases(...) __CPROVER_decreases(__VA_ARGS__) +/* cassert to avoid confusion with in-built assert */ +#define cassert(x) __CPROVER_assert(x, "cbmc assertion failed") +#define assume(...) __CPROVER_assume(__VA_ARGS__) + +/*************************************************** + * Macros for "expression" forms that may appear + * _inside_ top-level contracts. + ***************************************************/ + +/* + * function return value - useful inside ensures + * https://diffblue.github.io/cbmc/contracts-functions.html + */ +#define return_value (__CPROVER_return_value) + +/* + * assigns l-value targets + * https://diffblue.github.io/cbmc/contracts-assigns.html + */ +#define object_whole(...) __CPROVER_object_whole(__VA_ARGS__) +#define memory_slice(...) __CPROVER_object_upto(__VA_ARGS__) + +/* + * Pointer-related predicates + * https://diffblue.github.io/cbmc/contracts-memory-predicates.html + */ +#define memory_no_alias(...) __CPROVER_is_fresh(__VA_ARGS__) +#define readable(...) __CPROVER_r_ok(__VA_ARGS__) +#define writeable(...) __CPROVER_w_ok(__VA_ARGS__) + +/* Maximum supported buffer size + * + * Larger buffers may be supported, but due to internal modeling constraints + * in CBMC, the proofs of memory- and type-safety won't be able to run. + * + * If you find yourself in need for a buffer size larger than this, + * please contact the maintainers, so we can prioritize work to relax + * this somewhat artificial bound. + */ +#define MLK_MAX_BUFFER_SIZE (SIZE_MAX >> 12) + +/* + * History variables + * https://diffblue.github.io/cbmc/contracts-history-variables.html + */ +#define old(...) __CPROVER_old(__VA_ARGS__) +#define loop_entry(...) __CPROVER_loop_entry(__VA_ARGS__) + +/* + * Quantifiers + * Note that the range on qvar is _exclusive_ between qvar_lb .. qvar_ub + * https://diffblue.github.io/cbmc/contracts-quantifiers.html + * + * The quantified variable is declared as uint32_t, so these macros + * quantify only over indices in [0, UINT32_MAX). Bounds larger than + * UINT32_MAX (4 GiB) are NOT supported: the explicit (uint32_t) casts + * on the bounds will trigger CBMC's conversion check if a wider bound + * (e.g. a size_t > UINT32_MAX) is passed. + * + * Quantifying over size_t (64-bit) was found to blow up SMT proof + * times, so we deliberately keep the index width at 32 bits. Callers + * dealing with size_t-typed buffers must add an explicit + * requires(len <= UINT32_MAX) + * precondition. + */ + +/* + * Prevent clang-format from corrupting CBMC's special ==> operator + */ +/* clang-format off */ +#define forall(qvar, qvar_lb, qvar_ub, predicate) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> (predicate) \ + } + +#define exists(qvar, qvar_lb, qvar_ub, predicate) \ + __CPROVER_exists \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) && (predicate) \ + } +/* clang-format on */ + +/*************************************************** + * Convenience macros for common contract patterns + ***************************************************/ + +/* + * Boolean-value predidate that asserts that "all values of array_var are in + * range value_lb (inclusive) .. value_ub (exclusive)" + * Example: + * array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q) + * expands to + * __CPROVER_forall { int k; (0 <= k && k <= MLKEM_N-1) ==> ( + * 0 <= a->coeffs[k]) && a->coeffs[k] < MLKEM_Q)) } + */ + +/* + * Prevent clang-format from corrupting CBMC's special ==> operator + */ +/* clang-format off */ +#define CBMC_CONCAT_(left, right) left##right +#define CBMC_CONCAT(left, right) CBMC_CONCAT_(left, right) + +#define array_bound_core(qvar, qvar_lb, qvar_ub, array_var, \ + value_lb, value_ub) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + (((int)(value_lb) <= ((array_var)[(qvar)])) && \ + (((array_var)[(qvar)]) < (int)(value_ub))) \ + } + +#define array_bound(array_var, qvar_lb, qvar_ub, value_lb, value_ub) \ + array_bound_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (qvar_lb), \ + (qvar_ub), (array_var), (value_lb), (value_ub)) + +#define array_unchanged_core(qvar, qvar_lb, qvar_ub, array_var) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + ((array_var)[(qvar)]) == (old(* (int16_t (*)[(qvar_ub)])(array_var)))[(qvar)] \ + } + +#define array_unchanged(array_var, N) \ + array_unchanged_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var)) + +#define array_unchanged_u64_core(qvar, qvar_lb, qvar_ub, array_var) \ + __CPROVER_forall \ + { \ + uint32_t qvar; \ + ((uint32_t) (qvar_lb) <= (qvar) && (qvar) < (uint32_t) (qvar_ub)) ==> \ + ((array_var)[(qvar)]) == (old(* (uint64_t (*)[(qvar_ub)])(array_var)))[(qvar)] \ + } + +#define array_unchanged_u64(array_var, N) \ + array_unchanged_u64_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), 0, (N), (array_var)) + +#define array_unchanged_u8_core(qvar, array_var, N) \ + forall(qvar, 0, (N), \ + ((array_var)[(qvar)]) == (old(* (uint8_t (*)[(N)])(array_var)))[(qvar)]) + +#define array_unchanged_u8(array_var, N) \ + array_unchanged_u8_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (array_var), (N)) + +#define array_zeroized_u8_core(qvar, array_var, N) \ + forall(qvar, 0, (N), ((array_var)[(qvar)]) == 0) + +#define array_zeroized_u8(array_var, N) \ + array_zeroized_u8_core(CBMC_CONCAT(_cbmc_idx, __COUNTER__), (array_var), (N)) +/* clang-format on */ + +/* + * Output-buffer discipline on failure, as documented in API-CONVENTIONS.md: + * when a function fails, each caller-owned output buffer is left either + * fully unchanged or fully zeroized -- never holding partially computed or + * stale data that could be mistaken for a valid result. + * + * Note the disjunction is over the buffer as a whole: it is not enough for + * each byte to be individually either unchanged or zero. + */ +#define array_unchanged_or_zeroized_u8(array_var, N) \ + (array_unchanged_u8((array_var), (N)) || array_zeroized_u8((array_var), (N))) + +/* Wrapper around array_bound operating on absolute values. + * + * The absolute value bound `k` is exclusive. + * + * Note that since the lower bound in array_bound is inclusive, we have to + * raise it by 1 here. + */ +#define array_abs_bound(arr, lb, ub, k) \ + array_bound((arr), (lb), (ub), -((int)(k)) + 1, (k)) + +#endif /* CBMC */ + +#endif /* !MLK_CBMC_H */ diff --git a/src/mlkem-native/mlkem/src/common.h b/src/mlkem-native/mlkem/src/common.h new file mode 100644 index 0000000000..96e0676fee --- /dev/null +++ b/src/mlkem-native/mlkem/src/common.h @@ -0,0 +1,296 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_COMMON_H +#define MLK_COMMON_H + +#ifndef __ASSEMBLER__ +#include +#endif + +#define MLK_BUILD_INTERNAL + +#if defined(MLK_CONFIG_FILE) +#include MLK_CONFIG_FILE +#else +#include "mlkem_native_config.h" +#endif + +#include "params.h" +#include "sys.h" + +/* Internal and public API have external linkage by default, but + * this can be overwritten by the user, e.g. for single-CU builds. */ +#if !defined(MLK_CONFIG_INTERNAL_API_QUALIFIER) +#define MLK_INTERNAL_API +#define MLK_INTERNAL_DATA_DECLARATION extern +#define MLK_INTERNAL_DATA_DEFINITION +#else +#define MLK_INTERNAL_API MLK_CONFIG_INTERNAL_API_QUALIFIER +#define MLK_INTERNAL_DATA_DECLARATION MLK_CONFIG_INTERNAL_API_QUALIFIER +#define MLK_INTERNAL_DATA_DEFINITION MLK_CONFIG_INTERNAL_API_QUALIFIER +#endif + +#if !defined(MLK_CONFIG_EXTERNAL_API_QUALIFIER) +#define MLK_EXTERNAL_API +#else +#define MLK_EXTERNAL_API MLK_CONFIG_EXTERNAL_API_QUALIFIER +#endif + +#define MLK_CONCAT_(x1, x2) x1##x2 +#define MLK_CONCAT(x1, x2) MLK_CONCAT_(x1, x2) + +#if (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || \ + defined(MLK_CONFIG_MULTILEVEL_NO_SHARED)) +#define MLK_ADD_PARAM_SET(s) MLK_CONCAT(s, MLK_CONFIG_PARAMETER_SET) +#else +#define MLK_ADD_PARAM_SET(s) s +#endif + +#define MLK_NAMESPACE_PREFIX MLK_CONCAT(MLK_CONFIG_NAMESPACE_PREFIX, _) +#define MLK_NAMESPACE_PREFIX_K \ + MLK_CONCAT(MLK_ADD_PARAM_SET(MLK_CONFIG_NAMESPACE_PREFIX), _) + +/* Functions are prefixed by MLK_CONFIG_NAMESPACE_PREFIX. + * + * If multiple parameter sets are used, functions depending on the parameter + * set are additionally prefixed with 512/768/1024. See mlkem_native_config.h. + * + * Example: If MLK_CONFIG_NAMESPACE_PREFIX is mlkem, then + * MLK_NAMESPACE_K(enc) becomes mlkem512_enc/mlkem768_enc/mlkem1024_enc. + */ +#define MLK_NAMESPACE(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX, s) +#define MLK_NAMESPACE_K(s) MLK_CONCAT(MLK_NAMESPACE_PREFIX_K, s) + +/* On Apple platforms, we need to emit leading underscore + * in front of assembly symbols. We thus introduce a separate + * namespace wrapper for ASM symbols. */ +#if !defined(__APPLE__) +#define MLK_ASM_NAMESPACE(sym) MLK_NAMESPACE(sym) +#else +#define MLK_ASM_NAMESPACE(sym) MLK_CONCAT(_, MLK_NAMESPACE(sym)) +#endif + +/* + * On X86_64 if control-flow protections (CET) are enabled (through + * -fcf-protection=), we add an endbr64 instruction at every global function + * label. See sys.h for more details + */ +#if defined(MLK_SYS_X86_64) +#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) : MLK_CET_ENDBR +#elif defined(MLK_SYS_ARMV81M_MVE) +/* clang-format off */ +#define MLK_ASM_FN_SYMBOL(sym) \ + .type MLK_ASM_NAMESPACE(sym), %function; \ + MLK_ASM_NAMESPACE(sym) : +/* clang-format on */ +#else /* !MLK_SYS_X86_64 && MLK_SYS_ARMV81M_MVE */ +#define MLK_ASM_FN_SYMBOL(sym) MLK_ASM_NAMESPACE(sym) : +#endif /* !MLK_SYS_X86_64 && !MLK_SYS_ARMV81M_MVE */ + +/* + * Output the size of an assembly function. + */ +#if defined(__ELF__) +#define MLK_ASM_FN_SIZE(sym) \ + .size MLK_ASM_NAMESPACE(sym), .- MLK_ASM_NAMESPACE(sym) +#else +#define MLK_ASM_FN_SIZE(sym) +#endif + +/* We aim to simplify the user's life by supporting builds where + * all source files are included, even those that are not needed. + * Those files are appropriately guarded and will be empty when unneeded. + * The following is to avoid compilers complaining about this. */ +#define MLK_EMPTY_CU(s) extern int MLK_NAMESPACE_K(empty_cu_##s); + +/* MLK_CONFIG_NO_ASM takes precedence over MLK_USE_NATIVE_XXX */ +#if defined(MLK_CONFIG_NO_ASM) +#undef MLK_CONFIG_USE_NATIVE_BACKEND_ARITH +#undef MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) && \ + !defined(MLK_CONFIG_ARITH_BACKEND_FILE) +#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_ARITH is set, but MLK_CONFIG_ARITH_BACKEND_FILE is not. +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) && \ + !defined(MLK_CONFIG_FIPS202_BACKEND_FILE) +#error Bad configuration: MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 is set, but MLK_CONFIG_FIPS202_BACKEND_FILE is not. +#endif + +#if defined(MLK_CONFIG_NO_RANDOMIZED_API) && defined(MLK_CONFIG_KEYGEN_PCT) +#error Bad configuration: MLK_CONFIG_NO_RANDOMIZED_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires enc() +#endif + +#if defined(MLK_CONFIG_NO_ENCAPS_API) && defined(MLK_CONFIG_KEYGEN_PCT) +#error Bad configuration: MLK_CONFIG_NO_ENCAPS_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires enc() +#endif + +#if defined(MLK_CONFIG_NO_DECAPS_API) && defined(MLK_CONFIG_KEYGEN_PCT) +#error Bad configuration: MLK_CONFIG_NO_DECAPS_API is incompatible with MLK_CONFIG_KEYGEN_PCT as the current PCT implementation requires dec() +#endif + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_ARITH) +#include MLK_CONFIG_ARITH_BACKEND_FILE +/* Include to enforce consistency of API and implementation, + * and conduct sanity checks on the backend. + * + * Keep this _after_ the inclusion of the backend; otherwise, + * the sanity checks won't have an effect. */ +#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__) +#include "native/api.h" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_ARITH */ + +#if defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) +#include MLK_CONFIG_FIPS202_BACKEND_FILE +/* Include to enforce consistency of API and implementation, + * and conduct sanity checks on the backend. + * + * Keep this _after_ the inclusion of the backend; otherwise, + * the sanity checks won't have an effect. */ +#if defined(MLK_CHECK_APIS) && !defined(__ASSEMBLER__) +#include "fips202/native/api.h" +#endif +#endif /* MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 */ + +#if !defined(MLK_CONFIG_FIPS202_CUSTOM_HEADER) +#define MLK_FIPS202_HEADER_FILE "fips202/fips202.h" +#else +#define MLK_FIPS202_HEADER_FILE MLK_CONFIG_FIPS202_CUSTOM_HEADER +#endif + +#if !defined(MLK_CONFIG_FIPS202X4_CUSTOM_HEADER) +#define MLK_FIPS202X4_HEADER_FILE "fips202/fips202x4.h" +#else +#define MLK_FIPS202X4_HEADER_FILE MLK_CONFIG_FIPS202X4_CUSTOM_HEADER +#endif + +/* Standard library function replacements */ +#if !defined(__ASSEMBLER__) +#if !defined(MLK_CONFIG_CUSTOM_MEMCPY) +#include +#define mlk_memcpy memcpy +#endif + +#if !defined(MLK_CONFIG_CUSTOM_MEMSET) +#include +#define mlk_memset memset +#endif + + +/* Allocation macros for large local structures + * + * MLK_ALLOC(v, T, N) declares T *v and attempts to point it to an T[N] + * MLK_FREE(v, T, N) zeroizes and frees the allocation + * + * Default implementation uses stack allocation. + * Can be overridden by setting the config option MLK_CONFIG_CUSTOM_ALLOC_FREE + * and defining MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE. + */ +#if defined(MLK_CONFIG_CUSTOM_ALLOC_FREE) != \ + (defined(MLK_CUSTOM_ALLOC) && defined(MLK_CUSTOM_FREE)) +#error Bad configuration: MLK_CONFIG_CUSTOM_ALLOC_FREE must be set together with MLK_CUSTOM_ALLOC and MLK_CUSTOM_FREE +#endif + +/* Context-parameter machinery (MLK_CONTEXT_PARAMETERS_n and related config + * checks). Kept in a separate, level-generic header for readability; included + * here so it is available to the allocation macros below and to all consumers + * of common.h. */ +#include "context.h" + +#if !defined(MLK_CONFIG_CUSTOM_ALLOC_FREE) +/* Default: stack allocation */ + +/* This is a declaration macro, not an expression macro: T is a type and v is + * a declarator, neither of which can be wrapped in parentheses. The + * bugprone-macro-parentheses diagnostic is therefore a false positive here. */ +#define MLK_ALLOC(v, T, N, context) \ + MLK_ALIGN T mlk_alloc_##v[N]; \ + T *v = mlk_alloc_##v /* NOLINT(bugprone-macro-parentheses) */ + +/* The MLK_FREE macro body references mlk_zeroize(), which is declared in + * verify.h. We deliberately do NOT include verify.h here: doing so would + * create a circular dependency (verify.h includes common.h), and common.h + * itself never calls mlk_zeroize() -- only the macro expansion does. Each + * translation unit that uses MLK_FREE therefore includes verify.h directly. */ +#define MLK_FREE(v, T, N, context) \ + do \ + { \ + MLK_CONTEXT_UNUSED(context); \ + mlk_zeroize(mlk_alloc_##v, sizeof(mlk_alloc_##v)); \ + (v) = NULL; \ + } while (0) + +#else /* !MLK_CONFIG_CUSTOM_ALLOC_FREE */ + +/* Custom allocation */ + +/* + * The indirection here is necessary to use MLK_CONTEXT_PARAMETERS_3 here. + */ +#define MLK_APPLY(f, args) f args + +#define MLK_ALLOC(v, T, N, context) \ + MLK_APPLY(MLK_CUSTOM_ALLOC, MLK_CONTEXT_PARAMETERS_3(v, T, N, context)) + +#define MLK_FREE(v, T, N, context) \ + do \ + { \ + if (v != NULL) \ + { \ + mlk_zeroize(v, sizeof(T) * (N)); \ + MLK_APPLY(MLK_CUSTOM_FREE, MLK_CONTEXT_PARAMETERS_3(v, T, N, context)); \ + v = NULL; \ + } \ + } while (0) + +#endif /* MLK_CONFIG_CUSTOM_ALLOC_FREE */ + +/****************************** Error codes ***********************************/ + +/* Generic failure condition. Currently not returned by any function; + * reserved for failures that no more specific code covers. */ +#define MLK_ERR_FAIL (-1) +/* An allocation failed. This can only happen if MLK_CONFIG_CUSTOM_ALLOC_FREE + * is defined and the provided MLK_CUSTOM_ALLOC can fail. */ +#define MLK_ERR_OUT_OF_MEMORY (-2) +/* An RNG failure occurred. Might be due to insufficient entropy or + * system misconfiguration. */ +#define MLK_ERR_RNG_FAIL (-3) +/* Public key validation failed: the @[FIPS203, Section 7.2, 'modulus check'] + * found a coefficient outside [0,q-1]. Returned by check_pk and by the + * encapsulation API. */ +#define MLK_ERR_INVALID_PK (-4) +/* Secret key validation failed: the @[FIPS203, Section 7.3, 'hash check'] + * found the embedded public key hash inconsistent. Returned by check_sk and + * by the decapsulation API. */ +#define MLK_ERR_INVALID_SK (-5) +/* The 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and + * @[FIPS203, Section 7.1, Pairwise Consistency] failed. Only possible when + * MLK_CONFIG_KEYGEN_PCT is enabled; signals that the freshly generated key + * pair failed its encaps/decaps self-test. */ +#define MLK_ERR_PCT_FAIL (-6) + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_COMMON_H */ diff --git a/src/mlkem-native/mlkem/src/compress.c b/src/mlkem-native/mlkem/src/compress.c new file mode 100644 index 0000000000..b42c2584fa --- /dev/null +++ b/src/mlkem-native/mlkem/src/compress.c @@ -0,0 +1,763 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "cbmc.h" +#include "compress.h" +#include "debug.h" +#include "verify.h" + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \ + MLKEM_K == 3) +/* Reference: `poly_compress()` in the reference implementation @[REF], + * for ML-KEM-{512,768}. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d4_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8] = {0}; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_bound(t, 0, j, 0, 16)) + decreases(8 - j)) + { + t[j] = mlk_scalar_compress_d4(a->coeffs[8 * i + j]); + } + + /* All t[i] are 4-bit wide, so the truncations don't alter the value. */ + r[i * 4] = (uint8_t)(t[0] | (t[1] << 4)); + r[i * 4 + 1] = (uint8_t)(t[2] | (t[3] << 4)); + r[i * 4 + 2] = (uint8_t)(t[4] | (t[5] << 4)); + r[i * 4 + 3] = (uint8_t)(t[6] | (t[7] << 4)); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d4_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */ + + mlk_poly_compress_d4_c(r, a); +} + +/* Reference: Embedded into `polyvec_compress()` in the + * reference implementation, for ML-KEM-{512,768}. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d10_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +) +{ + unsigned j; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + for (j = 0; j < MLKEM_N / 4; j++) + __loop__(invariant(j <= MLKEM_N / 4) + decreases(MLKEM_N / 4 - j)) + { + unsigned k; + uint16_t t[4]; + for (k = 0; k < 4; k++) + __loop__( + invariant(k <= 4) + invariant(forall(r, 0, k, t[r] < (1u << 10))) + decreases(4 - k)) + { + t[k] = mlk_scalar_compress_d10(a->coeffs[4 * j + k]); + } + + /* + * Make all implicit truncation explicit. No data is being + * truncated for the LHS's since each t[i] is 10-bit in size. + */ + r[5 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF); + r[5 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 2) & 0xFF)); + r[5 * j + 2] = (uint8_t)((t[1] >> 6) | ((t[2] << 4) & 0xFF)); + r[5 * j + 3] = (uint8_t)((t[2] >> 4) | ((t[3] << 6) & 0xFF)); + r[5 * j + 4] = (uint8_t)(t[3] >> 2); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d10_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */ + + mlk_poly_compress_d10_c(r, a); +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_decompress()` in the reference implementation @[REF], + * for ML-KEM-{512,768}. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d4_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 2; i++) + __loop__( + invariant(i <= MLKEM_N / 2) + invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 2 - i)) + { + r->coeffs[2 * i + 0] = mlk_scalar_decompress_d4((a[i] >> 0) & 0xF); + r->coeffs[2 * i + 1] = mlk_scalar_decompress_d4((a[i] >> 4) & 0xF); + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d4(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4) + int ret; + ret = mlk_poly_decompress_d4_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */ + + mlk_poly_decompress_d4_c(r, a); +} + +/* Reference: Embedded into `polyvec_decompress()` in the + * reference implementation, for ML-KEM-{512,768}. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d10_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned j; + for (j = 0; j < MLKEM_N / 4; j++) + __loop__( + invariant(j <= MLKEM_N / 4) + invariant(array_bound(r->coeffs, 0, 4 * j, 0, MLKEM_Q)) + decreases(MLKEM_N / 4 - j)) + { + unsigned k; + uint16_t t[4]; + uint8_t const *base = &a[5 * j]; + + t[0] = 0x3FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8)); + t[1] = 0x3FF & ((base[1] >> 2) | ((uint16_t)base[2] << 6)); + t[2] = 0x3FF & ((base[2] >> 4) | ((uint16_t)base[3] << 4)); + t[3] = 0x3FF & ((base[3] >> 6) | ((uint16_t)base[4] << 2)); + + for (k = 0; k < 4; k++) + __loop__( + invariant(k <= 4) + invariant(array_bound(r->coeffs, 0, 4 * j + k, 0, MLKEM_Q)) + decreases(4 - k)) + { + r->coeffs[4 * j + k] = mlk_scalar_decompress_d10(t[k]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d10(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10) + int ret; + ret = mlk_poly_decompress_d10_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */ + + mlk_poly_decompress_d10_c(r, a); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/* Reference: `poly_compress()` in the reference implementation @[REF], + * for ML-KEM-1024. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d5_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8] = {0}; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_bound(t, 0, j, 0, 32)) + decreases(8 - j)) + { + t[j] = mlk_scalar_compress_d5(a->coeffs[8 * i + j]); + } + + r[i * 5] = (uint8_t)(0xFF & ((t[0] >> 0) | (t[1] << 5))); + r[i * 5 + 1] = (uint8_t)(0xFF & ((t[1] >> 3) | (t[2] << 2) | (t[3] << 7))); + r[i * 5 + 2] = (uint8_t)(0xFF & ((t[3] >> 1) | (t[4] << 4))); + r[i * 5 + 3] = (uint8_t)(0xFF & ((t[4] >> 4) | (t[5] << 1) | (t[6] << 6))); + r[i * 5 + 4] = (uint8_t)(0xFF & ((t[6] >> 2) | (t[7] << 3))); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d5_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */ + + mlk_poly_compress_d5_c(r, a); +} + +/* Reference: Embedded into `polyvec_compress()` in the + * reference implementation, for ML-KEM-1024. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_compress_d11_c( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +) +{ + unsigned j; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (j = 0; j < MLKEM_N / 8; j++) + __loop__(invariant(j <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - j)) + { + unsigned k; + uint16_t t[8]; + for (k = 0; k < 8; k++) + __loop__( + invariant(k <= 8) + invariant(forall(r, 0, k, t[r] < (1u << 11))) + decreases(8 - k)) + { + t[k] = mlk_scalar_compress_d11(a->coeffs[8 * j + k]); + } + + /* + * Make all implicit truncation explicit. No data is being + * truncated for the LHS's since each t[i] is 11-bit in size. + */ + r[11 * j + 0] = (uint8_t)((t[0] >> 0) & 0xFF); + r[11 * j + 1] = (uint8_t)((t[0] >> 8) | ((t[1] << 3) & 0xFF)); + r[11 * j + 2] = (uint8_t)((t[1] >> 5) | ((t[2] << 6) & 0xFF)); + r[11 * j + 3] = (uint8_t)((t[2] >> 2) & 0xFF); + r[11 * j + 4] = (uint8_t)((t[2] >> 10) | ((t[3] << 1) & 0xFF)); + r[11 * j + 5] = (uint8_t)((t[3] >> 7) | ((t[4] << 4) & 0xFF)); + r[11 * j + 6] = (uint8_t)((t[4] >> 4) | ((t[5] << 7) & 0xFF)); + r[11 * j + 7] = (uint8_t)((t[5] >> 1) & 0xFF); + r[11 * j + 8] = (uint8_t)((t[5] >> 9) | ((t[6] << 2) & 0xFF)); + r[11 * j + 9] = (uint8_t)((t[6] >> 6) | ((t[7] << 5) & 0xFF)); + r[11 * j + 10] = (uint8_t)(t[7] >> 3); + } +} + +MLK_INTERNAL_API +void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_compress_d11_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */ + + mlk_poly_compress_d11_c(r, a); +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_decompress()` in the reference implementation @[REF], + * for ML-KEM-1024. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d5_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint8_t t[8]; + const unsigned offset = i * 5; + /* + * Explicitly truncate to avoid warning about + * implicit truncation in CBMC and unwind loop for ease + * of proof. + */ + + /* + * Decompress 5 8-bit bytes (so 40 bits) into + * 8 5-bit values stored in t[] + */ + t[0] = 0x1F & (a[offset + 0] >> 0); + t[1] = 0x1F & ((a[offset + 0] >> 5) | (a[offset + 1] << 3)); + t[2] = 0x1F & (a[offset + 1] >> 2); + t[3] = 0x1F & ((a[offset + 1] >> 7) | (a[offset + 2] << 1)); + t[4] = 0x1F & ((a[offset + 2] >> 4) | (a[offset + 3] << 4)); + t[5] = 0x1F & (a[offset + 3] >> 1); + t[6] = 0x1F & ((a[offset + 3] >> 6) | (a[offset + 4] << 2)); + t[7] = 0x1F & (a[offset + 4] >> 3); + + /* and copy to the correct slice in r[] */ + for (j = 0; j < 8; j++) + __loop__( + invariant(j <= 8 && i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q)) + decreases(8 - j)) + { + r->coeffs[8 * i + j] = mlk_scalar_decompress_d5(t[j]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d5(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5) + int ret; + ret = mlk_poly_decompress_d5_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */ + + mlk_poly_decompress_d5_c(r, a); +} + +/* Reference: Embedded into `polyvec_decompress()` in the + * reference implementation, for ML-KEM-1024. */ +MLK_STATIC_TESTABLE void mlk_poly_decompress_d11_c( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned j; + for (j = 0; j < MLKEM_N / 8; j++) + __loop__( + invariant(j <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * j, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - j)) + { + unsigned k; + uint16_t t[8]; + uint8_t const *base = &a[11 * j]; + t[0] = 0x7FF & ((base[0] >> 0) | ((uint16_t)base[1] << 8)); + t[1] = 0x7FF & ((base[1] >> 3) | ((uint16_t)base[2] << 5)); + t[2] = 0x7FF & ((base[2] >> 6) | ((uint16_t)base[3] << 2) | + ((uint16_t)base[4] << 10)); + t[3] = 0x7FF & ((base[4] >> 1) | ((uint16_t)base[5] << 7)); + t[4] = 0x7FF & ((base[5] >> 4) | ((uint16_t)base[6] << 4)); + t[5] = 0x7FF & ((base[6] >> 7) | ((uint16_t)base[7] << 1) | + ((uint16_t)base[8] << 9)); + t[6] = 0x7FF & ((base[8] >> 2) | ((uint16_t)base[9] << 6)); + t[7] = 0x7FF & ((base[9] >> 5) | ((uint16_t)base[10] << 3)); + + for (k = 0; k < 8; k++) + __loop__( + invariant(k <= 8) + invariant(array_bound(r->coeffs, 0, 8 * j + k, 0, MLKEM_Q)) + decreases(8 - k)) + { + r->coeffs[8 * j + k] = mlk_scalar_decompress_d11(t[k]); + } + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_decompress_d11(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11) + int ret; + ret = mlk_poly_decompress_d11_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */ + + mlk_poly_decompress_d11_c(r, a); +} + +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: `poly_tobytes()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_STATIC_TESTABLE void mlk_poly_tobytes_c(uint8_t r[MLKEM_POLYBYTES], + const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +) +{ + unsigned i; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 2; i++) + __loop__(invariant(i <= MLKEM_N / 2) + decreases(MLKEM_N / 2 - i)) + { + /* The conversion to uint16_t is safe since we assume that + * the coefficients of `a` are non-negative. */ + const uint16_t t0 = (uint16_t)a->coeffs[2 * i]; + const uint16_t t1 = (uint16_t)a->coeffs[2 * i + 1]; + /* + * t0 and t1 are both < MLKEM_Q, so contain at most 12 bits each of + * significant data, so these can be packed into 24 bits or exactly + * 3 bytes, as follows. + */ + + /* Least significant bits 0 - 7 of t0. */ + r[3 * i + 0] = (uint8_t)(t0 & 0xFF); + + /* + * Most significant bits 8 - 11 of t0 become the least significant + * nibble of the second byte. The least significant 4 bits + * of t1 become the upper nibble of the second byte. + * + * The conversion to uint8_t does not alter the value. + */ + r[3 * i + 1] = (uint8_t)((t0 >> 8) | ((t1 << 4) & 0xF0)); + + /* Bits 4 - 11 of t1 become the third byte. The conversion to uint8_t + * does not alter the value because t1 is 12-bit wide. */ + r[3 * i + 2] = (uint8_t)(t1 >> 4); + } +} + +MLK_INTERNAL_API +void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a) +{ +#if defined(MLK_USE_NATIVE_POLY_TOBYTES) + int ret; + mlk_assert_bound(a, MLKEM_N, 0, MLKEM_Q); + ret = mlk_poly_tobytes_native(r, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_TOBYTES */ + + mlk_poly_tobytes_c(r, a); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_frombytes()` in the reference implementation @[REF]. */ +MLK_STATIC_TESTABLE void mlk_poly_frombytes_c(mlk_poly *r, + const uint8_t a[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 2; i++) + __loop__( + invariant(i <= MLKEM_N / 2) + invariant(array_bound(r->coeffs, 0, 2 * i, 0, MLKEM_UINT12_LIMIT)) + decreases(MLKEM_N / 2 - i)) + { + const uint8_t t0 = a[3 * i + 0]; + const uint8_t t1 = a[3 * i + 1]; + const uint8_t t2 = a[3 * i + 2]; + /* Safety: + * - The explicit cast to uint16_t ensures that << 8 does + * not signed-overflow even on a 16-bit system. + * - The cast to int16_t is safe due to the explicit 0xFFF truncation. + */ + r->coeffs[2 * i + 0] = (int16_t)(t0 | (((uint16_t)t1 << 8) & 0xFFF)); + r->coeffs[2 * i + 1] = (int16_t)((t1 >> 4) | (t2 << 4)); + } + + /* Note that the coefficients are not canonical */ + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +} + +MLK_INTERNAL_API +void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES]) +{ +#if defined(MLK_USE_NATIVE_POLY_FROMBYTES) + int ret; + ret = mlk_poly_frombytes_native(r->coeffs, a); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_FROMBYTES */ + + mlk_poly_frombytes_c(r, a); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_frommsg()` in the reference implementation @[REF]. + * - We use a value barrier around the bit-selection mask to + * reduce the risk of compiler-introduced branches. + * The reference implementation contains the expression + * `(msg[i] >> j) & 1` which the compiler can reason must + * be either 0 or 1. */ +MLK_INTERNAL_API +void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES]) +{ + unsigned i; +#if (MLKEM_INDCPA_MSGBYTES != MLKEM_N / 8) +#error "MLKEM_INDCPA_MSGBYTES must be equal to MLKEM_N/8 bytes!" +#endif + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_bound(r->coeffs, 0, 8 * i, 0, MLKEM_Q)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + for (j = 0; j < 8; j++) + __loop__( + invariant(i < MLKEM_N / 8 && j <= 8) + invariant(array_bound(r->coeffs, 0, 8 * i + j, 0, MLKEM_Q)) + decreases(8 - j)) + { + /* mlk_ct_sel_int16(MLKEM_Q_HALF, 0, b) is `Decompress_1(b != 0)` + * as per @[FIPS203, Eq (4.8)]. */ + + /* Prevent the compiler from recognizing this as a bit selection */ + uint8_t mask = mlk_value_barrier_u8((uint8_t)(1u << j)); + r->coeffs[8 * i + j] = mlk_ct_sel_int16(MLKEM_Q_HALF, 0, msg[i] & mask); + } + } + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_tomsg()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r) +{ + unsigned i; + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_N / 8; i++) + __loop__(invariant(i <= MLKEM_N / 8) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + msg[i] = 0; + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + decreases(8 - j)) + { + uint32_t t = mlk_scalar_compress_d1(r->coeffs[8 * i + j]); + msg[i] |= (uint8_t)(t << j); + } + } +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(compress) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/compress.h b/src/mlkem-native/mlkem/src/compress.h new file mode 100644 index 0000000000..19e1d402b2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/compress.h @@ -0,0 +1,613 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#ifndef MLK_COMPRESS_H +#define MLK_COMPRESS_H + + +#include "cbmc.h" +#include "common.h" +#include "debug.h" +#include "poly.h" +#include "verify.h" + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2 / MLKEM_Q). + * + * @spec{Compress_1 from @[FIPS203, Eq (4.7)].} + * + * @reference{Part of poly_tomsg() in the reference implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d1(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 2) + ensures(return_value == (((uint32_t)u * 2 + MLKEM_Q / 2) / MLKEM_Q) % 2) ) +{ + /* Compute as follows: + * ``` + * round(u * 2 / MLKEM_Q) + * = round(u * 2 * (2^31 / MLKEM_Q) / 2^31) + * ~= round(u * 2 * round(2^31 / MLKEM_Q) / 2^31) + * ``` + */ + /* check-magic: 1290168 == 2*round(2^31 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290168; + /* Unsigned shifting by 31 positions leaves only the top bit. */ + return (uint8_t)((d0 + ((uint32_t)1u << 30)) >> 31); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 16 / MLKEM_Q) % 16. + * + * @spec{Compress_4 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `poly_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d4(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 16) + ensures(return_value == (((uint32_t)u * 16 + MLKEM_Q / 2) / MLKEM_Q) % 16)) +{ + /* Compute as follows: + * ``` + * round(u * 16 / MLKEM_Q) + * = round(u * 16 * (2^28 / MLKEM_Q) / 2^28) + * ~= round(u * 16 * round(2^28 / MLKEM_Q) / 2^28) + * ``` + */ + /* check-magic: 1290160 == 16 * round(2^28 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290160; + /* The return value is < 16, so not altered by the conversion to uint8_t. */ + return (uint8_t)((d0 + ((uint32_t)1u << 27)) >> 28); /* round(d0/2^28) */ +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 16). + * + * @spec{Decompress_4 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `poly_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 16 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d4(uint8_t u) +__contract__( + requires(0 <= u && u < 16) + ensures(return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 8) >> 4); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 32 / MLKEM_Q) % 32. + * + * @spec{Compress_5 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `poly_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint8_t mlk_scalar_compress_d5(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < 32) + ensures(return_value == (((uint32_t)u * 32 + MLKEM_Q / 2) / MLKEM_Q) % 32) ) +{ + /* Compute as follows: + * ``` + * round(u * 32 / MLKEM_Q) + * = round(u * 32 * (2^27 / MLKEM_Q) / 2^27) + * ~= round(u * 32 * round(2^27 / MLKEM_Q) / 2^27) + * ``` + */ + /* check-magic: 1290176 == 2^5 * round(2^27 / MLKEM_Q) */ + uint32_t d0 = (uint32_t)u * 1290176; + /* The return value is < 32, so not altered by the conversion to uint8_t. */ + return (uint8_t)((d0 + ((uint32_t)1u << 26)) >> 27); /* round(d0/2^27) */ +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 32). + * + * @spec{Decompress_5 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `poly_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 32 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d5(uint8_t u) +__contract__( + requires(0 <= u && u < 32) + ensures(0 <= return_value && return_value <= MLKEM_Q - 1) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 16) >> 5); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2**10 / MLKEM_Q) % 2**10. + * + * @spec{Compress_10 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `polyvec_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint16_t mlk_scalar_compress_d10(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < (1u << 10)) + ensures(return_value == (((uint32_t)u * (1u << 10) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 10))) +{ + /* Compute as follows: + * ``` + * round(u * 1024 / MLKEM_Q) + * = round(u * 1024 * (2^33 / MLKEM_Q) / 2^33) + * ~= round(u * 1024 * round(2^33 / MLKEM_Q) / 2^33) + * ``` + */ + /* check-magic: 2642263040 == 2^10 * round(2^33 / MLKEM_Q) */ + uint64_t d0 = (uint64_t)u * 2642263040; + d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */ + return (d0 & 0x3FF); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 1024). + * + * @spec{Decompress_10 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `polyvec_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 1024 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d10(uint16_t u) +__contract__( + requires(0 <= u && u < 1024) + ensures(0 <= return_value && return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 512) >> 10); +} + +/* + * The multiplication in this routine will exceed UINT32_MAX + * and wrap around for large values of u. This is expected and required. + */ +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "unsigned-overflow" +#endif + +/** + * Compute round(u * 2**11 / MLKEM_Q) % 2**11. + * + * @spec{Compress_11 from @[FIPS203, Eq (4.7)].} + * + * @reference{Embedded into `polyvec_compress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo MLKEM_Q to be compressed. + * + * @return Compressed value. + */ +static MLK_INLINE uint16_t mlk_scalar_compress_d11(int16_t u) +__contract__( + requires(0 <= u && u <= MLKEM_Q - 1) + ensures(return_value < (1u << 11)) + ensures(return_value == (((uint32_t)u * (1u << 11) + MLKEM_Q / 2) / MLKEM_Q) % (1 << 11))) +{ + /* Compute as follows: + * ``` + * round(u * 2048 / MLKEM_Q) + * = round(u * 2048 * (2^33 / MLKEM_Q) / 2^33) + * ~= round(u * 2048 * round(2^33 / MLKEM_Q) / 2^33) + * ``` + */ + /* check-magic: 5284526080 == 2^11 * round(2^33 / MLKEM_Q) */ + uint64_t d0 = (uint64_t)u * 5284526080; + d0 = (d0 + ((uint64_t)1u << 32)) >> 33; /* round(d0/2^33) */ + return (d0 & 0x7FF); +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Compute round(u * MLKEM_Q / 2048). + * + * @spec{Decompress_11 from @[FIPS203, Eq (4.8)].} + * + * @reference{Embedded into `polyvec_decompress()` in the reference + * implementation @[REF].} + * + * @param u Unsigned canonical modulus modulo 2048 to be decompressed. + * + * @return Decompressed value. + */ +static MLK_INLINE int16_t mlk_scalar_decompress_d11(uint16_t u) +__contract__( + requires(0 <= u && u < 2048) + ensures(0 <= return_value && return_value <= (MLKEM_Q - 1)) +) +{ + /* The return value is in 0..MLKEM_Q-1, hence not altered by the + * conversion to int16_t. */ + return (int16_t)((((uint32_t)u * MLKEM_Q) + 1024) >> 11); +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +#define mlk_poly_compress_d4 MLK_NAMESPACE(poly_compress_d4) +/** + * Compression (4 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_4 (Compress_4 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L23], where `d_v=4` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d4(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const mlk_poly *a); + +#define mlk_poly_compress_d10 MLK_NAMESPACE(poly_compress_d10) +/** + * Compression (10 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_10 (Compress_10 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L22], where `d_u=10` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d10(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const mlk_poly *a); + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_d4 MLK_NAMESPACE(poly_decompress_d4) +/** + * De-serialization and subsequent decompression (4 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d4. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_4 (ByteDecode_4 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L4], where `d_v=4` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d4(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]); + +#define mlk_poly_decompress_d10 MLK_NAMESPACE(poly_decompress_d10) +/** + * De-serialization and subsequent decompression (10 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d10. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_10 (ByteDecode_10 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L3], where `d_u=10` for ML-KEM-{512,768} @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d10(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +#define mlk_poly_compress_d5 MLK_NAMESPACE(poly_compress_d5) +/** + * Compression (5 bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_5 (Compress_5 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_v} (Compress_{d_v} (v))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L23], where `d_v=5` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d5(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const mlk_poly *a); + +#define mlk_poly_compress_d11 MLK_NAMESPACE(poly_compress_d11) +/** + * Compression (11 bits) and subsequent serialization of a polynomial. + * + * @spec{`ByteEncode_11 (Compress_11 (a))`: ByteEncode_d @[FIPS203, + * Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to vectors as + * per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_{d_u} (Compress_{d_u} (u))` appears in @[FIPS203, Algorithm + * 14 (K-PKE.Encrypt), L22], where `d_u=11` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_compress_d11(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const mlk_poly *a); + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_d5 MLK_NAMESPACE(poly_decompress_d5) +/** + * De-serialization and subsequent decompression (5 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d5. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_5 (ByteDecode_5 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_v} (ByteDecode_{d_v} (v))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L4], where `d_v=5` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d5(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]); + +#define mlk_poly_decompress_d11 MLK_NAMESPACE(poly_decompress_d11) +/** + * De-serialization and subsequent decompression (11 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d11. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_11 (ByteDecode_11 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_{d_u} (ByteDecode_{d_u} (u))` appears in @[FIPS203, Algorithm + * 15 (K-PKE.Decrypt), L3], where `d_u=11` for ML-KEM-1024 @[FIPS203, + * Table 2].} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + */ +MLK_INTERNAL_API +void mlk_poly_decompress_d11(mlk_poly *r, + const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +#define mlk_poly_tobytes MLK_NAMESPACE(poly_tobytes) +/** + * Serialization of a polynomial. Signed coefficients are converted to + * unsigned form before serialization. + * + * @spec{Implements ByteEncode_12 @[FIPS203, Algorithm 5]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].} + * + * @param[out] r Output byte array (of MLKEM_POLYBYTES bytes). + * @param[in] a Input polynomial, with each coefficient in the range + * [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_poly_tobytes(uint8_t r[MLKEM_POLYBYTES], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_frombytes MLK_NAMESPACE(poly_frombytes) +/** + * De-serialization of a polynomial. + * + * @spec{Implements ByteDecode_12 @[FIPS203, Algorithm 6]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays].} + * + * @param[out] r Output polynomial, with each coefficient unsigned and in + * the range 0..4095. + * @param[in] a Input byte array (of MLKEM_POLYBYTES bytes). + */ +MLK_INTERNAL_API +void mlk_poly_frombytes(mlk_poly *r, const uint8_t a[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_frommsg MLK_NAMESPACE(poly_frommsg) +/** + * Convert a 32-byte message to a polynomial. + * + * @spec{Implements `Decompress_1 (ByteDecode_1 (a))`: ByteDecode_d + * @[FIPS203, Algorithm 6], Decompress_d @[FIPS203, Eq (4.8)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `Decompress_1 (ByteDecode_1 (w))` appears in @[FIPS203, Algorithm 15 + * (K-PKE.Encrypt), L20].} + * + * @param[out] r Output polynomial. + * @param[in] msg Input message. + */ +MLK_INTERNAL_API +void mlk_poly_frommsg(mlk_poly *r, const uint8_t msg[MLKEM_INDCPA_MSGBYTES]) +__contract__( + requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_tomsg MLK_NAMESPACE(poly_tomsg) +/** + * Convert a polynomial to a 32-byte message. + * + * @spec{Implements `ByteEncode_1 (Compress_1 (a))`: ByteEncode_d + * @[FIPS203, Algorithm 5], Compress_d @[FIPS203, Eq (4.7)], extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays]. + * `ByteEncode_1 (Compress_1 (w))` appears in @[FIPS203, Algorithm 14 + * (K-PKE.Decrypt), L7].} + * + * @param[out] msg Output message. + * @param[in] r Input polynomial. Coefficients must be unsigned canonical. + */ +MLK_INTERNAL_API +void mlk_poly_tomsg(uint8_t msg[MLKEM_INDCPA_MSGBYTES], const mlk_poly *r) +__contract__( + requires(memory_no_alias(msg, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(msg, MLKEM_INDCPA_MSGBYTES)) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_COMPRESS_H */ diff --git a/src/mlkem-native/mlkem/src/context.h b/src/mlkem-native/mlkem/src/context.h new file mode 100644 index 0000000000..9a07ef2846 --- /dev/null +++ b/src/mlkem-native/mlkem/src/context.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_CONTEXT_H +#define MLK_CONTEXT_H + +/* This header is included by common.h once the configuration has been pulled + * in; it is not meant to be included directly. */ + +/* + * If the integration wants to provide a context parameter for use in + * platform-specific hooks, then it should define this parameter. + * + * The MLK_CONTEXT_PARAMETERS_n macros are intended to be used with macros + * defining the function names and expand to either pass or discard the context + * argument as required by the current build. If there is no context parameter + * requested then these are removed from the prototypes and from all calls. + */ +#ifdef MLK_CONFIG_CONTEXT_PARAMETER +#define MLK_CONTEXT_PARAMETERS_0(context) (context) +#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0, context) +#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1, context) +#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) \ + (arg0, arg1, arg2, context) +#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3, context) +#else /* MLK_CONFIG_CONTEXT_PARAMETER */ +#define MLK_CONTEXT_PARAMETERS_0(context) () +#define MLK_CONTEXT_PARAMETERS_1(arg0, context) (arg0) +#define MLK_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1) +#define MLK_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) (arg0, arg1, arg2) +#define MLK_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3) +#endif /* !MLK_CONFIG_CONTEXT_PARAMETER */ + +/* Consume a context parameter carried only for the integration's benefit, + * avoiding -Wunused-parameter; expands to nothing when no context is + * configured. */ +#if defined(MLK_CONFIG_CONTEXT_PARAMETER) +#define MLK_CONTEXT_UNUSED(context) ((void)(context)) +#else +#define MLK_CONTEXT_UNUSED(context) ((void)0) +#endif + +#if defined(MLK_CONFIG_CONTEXT_PARAMETER_TYPE) != \ + defined(MLK_CONFIG_CONTEXT_PARAMETER) +#error MLK_CONFIG_CONTEXT_PARAMETER_TYPE must be defined if and only if MLK_CONFIG_CONTEXT_PARAMETER is defined +#endif + +#endif /* !MLK_CONTEXT_H */ diff --git a/src/mlkem-native/mlkem/src/debug.c b/src/mlkem-native/mlkem/src/debug.c new file mode 100644 index 0000000000..386f5264ff --- /dev/null +++ b/src/mlkem-native/mlkem/src/debug.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */ + +#include "common.h" + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG) + + +#include +#include +#include "debug.h" + +#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] " + +void mlk_debug_check_assert(const char *file, int line, const int val) +{ + if (val == 0) + { + fprintf(stderr, MLK_DEBUG_ERROR_HEADER "Assertion failed (value %d)\n", + file, line, val); + exit(1); + } +} + +void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr, + unsigned len, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + int err = 0; + unsigned i; + for (i = 0; i < len; i++) + { + int16_t val = ptr[i]; + if (!(val > lower_bound_exclusive && val < upper_bound_exclusive)) + { + fprintf( + stderr, + MLK_DEBUG_ERROR_HEADER + "Bounds assertion failed: Index %u, value %d out of bounds (%d,%d)\n", + file, line, i, (int)val, lower_bound_exclusive, + upper_bound_exclusive); + err = 1; + } + } + + if (err == 1) + { + exit(1); + } +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG */ + +MLK_EMPTY_CU(debug) + +#endif /* !(!MLK_CONFIG_MULTILEVEL_NO_SHARED && MLKEM_DEBUG) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_DEBUG_ERROR_HEADER diff --git a/src/mlkem-native/mlkem/src/debug.h b/src/mlkem-native/mlkem/src/debug.h new file mode 100644 index 0000000000..ce4a33a373 --- /dev/null +++ b/src/mlkem-native/mlkem/src/debug.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_DEBUG_H +#define MLK_DEBUG_H +#include "common.h" + +#if defined(MLKEM_DEBUG) + +/** + * Check debug assertion. + * + * Prints an error message to stderr and calls exit(1) on failure. + * + * @param[in] file Filename. + * @param line Line number. + * @param val Value asserted to be non-zero. + */ +#define mlk_debug_check_assert MLK_NAMESPACE(mlkem_debug_assert) +void mlk_debug_check_assert(const char *file, int line, const int val); + +/** + * Check whether values in an array of int16_t are within specified bounds. + * + * Prints an error message to stderr and calls exit(1) on failure. + * + * @param[in] file Filename. + * @param line Line number. + * @param[in] ptr Base of array to be checked. + * @param len Number of int16_t in @p ptr. + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds MLK_NAMESPACE(mlkem_debug_check_bounds) +void mlk_debug_check_bounds(const char *file, int line, const int16_t *ptr, + unsigned len, int lower_bound_exclusive, + int upper_bound_exclusive); + +/* Check assertion, calling exit() upon failure + * + * val: Value that's asserted to be non-zero + */ +#define mlk_assert(val) mlk_debug_check_assert(__FILE__, __LINE__, (val)) + +/* Check bounds in array of int16_t's + * ptr: Base of int16_t array; will be explicitly cast to int16_t*, + * so you may pass a byte-compatible type such as mlk_poly or mlk_polyvec. + * len: Number of int16_t in array + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + mlk_debug_check_bounds(__FILE__, __LINE__, (const int16_t *)(ptr), (len), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in array of int16_t's + * ptr: Base of array, expression of type int16_t* + * len: Number of int16_t in array + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + mlk_assert_bound((ptr), (len), (-(value_abs_bd) + 1), (value_abs_bd)) + +/* Version of bounds assertions for 2-dimensional arrays */ +#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \ + mlk_assert_bound((ptr), ((len0) * (len1)), (value_lb), (value_ub)) + +#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \ + mlk_assert_abs_bound((ptr), ((len0) * (len1)), (value_abs_bd)) + +/* When running CBMC, convert debug assertions into proof obligations */ +#elif defined(CBMC) +#include "cbmc.h" + +#define mlk_assert(val) cassert(val) + +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + cassert(array_bound(((int16_t *)(ptr)), 0, (len), (value_lb), (value_ub))) + +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + cassert(array_abs_bound(((int16_t *)(ptr)), 0, (len), (value_abs_bd))) + +/* Because of https://github.com/diffblue/cbmc/issues/8570, we can't + * just use a single flattened array_bound(...) here. */ +#define mlk_assert_bound_2d(ptr, M, N, value_lb, value_ub) \ + cassert(forall(kN, 0, (M), \ + array_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \ + (value_lb), (value_ub)))) + +#define mlk_assert_abs_bound_2d(ptr, M, N, value_abs_bd) \ + cassert(forall(kN, 0, (M), \ + array_abs_bound(&((int16_t (*)[(N)])(ptr))[kN][0], 0, (N), \ + (value_abs_bd)))) + +#else /* !MLKEM_DEBUG && CBMC */ + +#define mlk_assert(val) \ + do \ + { \ + } while (0) +#define mlk_assert_bound(ptr, len, value_lb, value_ub) \ + do \ + { \ + } while (0) +#define mlk_assert_abs_bound(ptr, len, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_2d(ptr, len0, len1, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_2d(ptr, len0, len1, value_abs_bd) \ + do \ + { \ + } while (0) + + +#endif /* !MLKEM_DEBUG && !CBMC */ +#endif /* !MLK_DEBUG_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202.c b/src/mlkem-native/mlkem/src/fips202/fips202.c new file mode 100644 index 0000000000..33883aa9f8 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [mupq] + * Common files for pqm4, pqm3, pqriscv + * Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen + * https://github.com/mupq/mupq + * + * - [supercop] + * SUPERCOP benchmarking framework + * Daniel J. Bernstein + * http://bench.cr.yp.to/supercop.html + * + * - [tweetfips] + * 'tweetfips202' FIPS202 implementation + * Van Assche, Bernstein, Schwabe + * https://keccak.team/2015/tweetfips202.html + */ + +/* Based on the CC0 implementation from @[mupq] and the public domain + * implementation @[supercop, crypto_hash/keccakc512/simple/] + * by Ronny Van Keer, and the public domain @[tweetfips] implementation. */ + +#include "../common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "../verify.h" +#include "fips202.h" +#include "keccakf1600.h" + +/** + * Absorb step of Keccak; non-incremental, starts by zeroeing the state. + * + * @warning Must only be called once. + * + * @param[out] s Pointer to (uninitialized) output Keccak state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + * @param[in] m Input to be absorbed into @p s. + * @param mlen Length of input in bytes. + * @param p Domain-separation byte for different Keccak-derived + * functions. + */ +static void mlk_keccak_absorb_once(uint64_t *s, unsigned r, const uint8_t *m, + size_t mlen, uint8_t p) +__contract__( + requires(mlen <= MLK_MAX_BUFFER_SIZE) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(m, mlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES))) +{ + /* Initialize state */ + size_t i; + for (i = 0; i < 25; ++i) + __loop__(invariant(i <= 25) + decreases(25 - i)) + { + s[i] = 0; + } + + while (mlen >= r) + __loop__( + assigns(mlen, m, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + invariant(mlen <= loop_entry(mlen)) + invariant(m == loop_entry(m) + (loop_entry(mlen) - mlen)) + decreases(mlen)) + { + mlk_keccakf1600_xor_bytes(s, m, 0, r); + mlk_keccakf1600_permute(s); + mlen -= r; + m += r; + } + + /* At this point, mlen < r, so the truncations to unsigned are safe below. */ + + if (mlen > 0) + { + mlk_keccakf1600_xor_bytes(s, m, 0, (unsigned int)mlen); + } + + if (mlen == r - 1) + { + p |= 128; + mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1); + } + else + { + mlk_keccakf1600_xor_bytes(s, &p, (unsigned int)mlen, 1); + p = 128; + mlk_keccakf1600_xor_bytes(s, &p, r - 1, 1); + } +} + +/** + * Block-level Keccak squeeze. + * + * @param[out] h Output bytes. + * @param nblocks Number of blocks to be squeezed. + * @param[in,out] s Input/output state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + */ +static void mlk_keccak_squeezeblocks(uint8_t *h, size_t nblocks, uint64_t *s, + unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(h, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(h, nblocks * r))) +{ + while (nblocks > 0) + __loop__( + assigns(h, nblocks, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES), + memory_slice(h, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks) && + h == loop_entry(h) + r * (loop_entry(nblocks) - nblocks)) + decreases(nblocks)) + { + mlk_keccakf1600_permute(s); + mlk_keccakf1600_extract_bytes(s, h, 0, r); + h += r; + nblocks--; + } +} + +/** + * Keccak squeeze; can be called on byte-level. + * + * @warning Must only be called once. + * + * @param[out] h Output bytes. + * @param outlen Number of bytes to be squeezed. + * @param[in,out] s Keccak state. + * @param r Rate in bytes (e.g., 168 for SHAKE128). + */ +static void mlk_keccak_squeeze_once(uint8_t *h, size_t outlen, uint64_t *s, + unsigned r) +__contract__( + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(h, outlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(h, outlen))) +{ + size_t len; + while (outlen > 0) + __loop__( + assigns(len, h, outlen, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES), + memory_slice(h, outlen)) + invariant(outlen <= loop_entry(outlen) && + h == loop_entry(h) + (loop_entry(outlen) - outlen)) + decreases(outlen)) + { + mlk_keccakf1600_permute(s); + + if (outlen < r) + { + len = outlen; + } + else + { + len = r; + } + mlk_keccakf1600_extract_bytes(s, h, 0, (unsigned int)len); + h += len; + outlen -= len; + } +} + +void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input, + size_t inlen) +{ + mlk_keccak_absorb_once(state->ctx, SHAKE128_RATE, input, inlen, 0x1F); +} + +void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks, + mlk_shake128ctx *state) +{ + mlk_keccak_squeezeblocks(output, nblocks, state->ctx, SHAKE128_RATE); +} + +void mlk_shake128_init(mlk_shake128ctx *state) { (void)state; } +void mlk_shake128_release(mlk_shake128ctx *state) +{ + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(state, sizeof(mlk_shake128ctx)); +} + +typedef mlk_shake128ctx mlk_shake256ctx; +void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input, + size_t inlen) +{ + mlk_shake256ctx state; + /* Absorb input */ + mlk_keccak_absorb_once(state.ctx, SHAKE256_RATE, input, inlen, 0x1F); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, outlen, state.ctx, SHAKE256_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(&state, sizeof(state)); +} + +void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen) +{ + uint64_t ctx[25]; + /* Absorb input */ + mlk_keccak_absorb_once(ctx, SHA3_256_RATE, input, inlen, 0x06); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, 32, ctx, SHA3_256_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(ctx, sizeof(ctx)); +} + +void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen) +{ + uint64_t ctx[25]; + /* Absorb input */ + mlk_keccak_absorb_once(ctx, SHA3_512_RATE, input, inlen, 0x06); + /* Squeeze output */ + mlk_keccak_squeeze_once(output, 64, ctx, SHA3_512_RATE); + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(ctx, sizeof(ctx)); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202.h b/src/mlkem-native/mlkem/src/fips202/fips202.h new file mode 100644 index 0000000000..99eaccc0fe --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_FIPS202_H +#define MLK_FIPS202_FIPS202_H + +#include "../cbmc.h" +#include "../common.h" + +#define SHAKE128_RATE 168 +#define SHAKE256_RATE 136 +#define SHA3_256_RATE 136 +#define SHA3_384_RATE 104 +#define SHA3_512_RATE 72 + +/** Context for the non-incremental SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[25]; /**< Keccak state. */ +} MLK_ALIGN mlk_shake128ctx; + +#define mlk_shake128_absorb_once MLK_NAMESPACE(shake128_absorb_once) +/** + * One-shot absorb step of the SHAKE128 XOF. + * + * For call-sites (in mlkem-native): + * - This function MUST ONLY be called straight after mlk_shake128_init(). + * - This function MUST ONLY be called once. + * + * Consequently, for providers of custom FIPS202 code to be used with + * mlkem-native: + * - You may assume that the input context is freshly initialized via + * mlk_shake128_init(). + * - You may assume that this function is called exactly once. + * + * @param[in,out] state SHAKE128 context. + * @param[in] input Input to be absorbed into the state. + * @param inlen Length of input in bytes. + */ +void mlk_shake128_absorb_once(mlk_shake128ctx *state, const uint8_t *input, + size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mlk_shake128ctx))) + requires(memory_no_alias(input, inlen)) + assigns(memory_slice(state, sizeof(mlk_shake128ctx))) +); + +#define mlk_shake128_squeezeblocks MLK_NAMESPACE(shake128_squeezeblocks) +/** + * Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes + * each. Modifies the state. Can be called multiple times to keep squeezing, + * i.e., is incremental. + * + * @param[out] output Output blocks. + * @param nblocks Number of blocks to be squeezed (written to output). + * @param[in,out] state Keccak state. + */ +void mlk_shake128_squeezeblocks(uint8_t *output, size_t nblocks, + mlk_shake128ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mlk_shake128ctx))) + requires(memory_no_alias(output, nblocks * SHAKE128_RATE)) + assigns(memory_slice(output, nblocks * SHAKE128_RATE), memory_slice(state, sizeof(mlk_shake128ctx))) +); + +#define mlk_shake128_init MLK_NAMESPACE(shake128_init) +void mlk_shake128_init(mlk_shake128ctx *state); + +#define mlk_shake128_release MLK_NAMESPACE(shake128_release) +void mlk_shake128_release(mlk_shake128ctx *state); + +/* One-stop SHAKE256 call. Aliasing between input and + * output is not permitted */ +#define mlk_shake256 MLK_NAMESPACE(shake256) +/** + * SHAKE256 XOF with non-incremental API. + * + * @param[out] output Output buffer. + * @param outlen Requested output length in bytes. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_shake256(uint8_t *output, size_t outlen, const uint8_t *input, + size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, outlen)) + assigns(memory_slice(output, outlen)) +); + +/* One-stop SHA3_256 call. Aliasing between input and + * output is not permitted */ +#define SHA3_256_HASHBYTES 32 +#define mlk_sha3_256 MLK_NAMESPACE(sha3_256) +/** + * SHA3-256 with non-incremental API. + * + * @param[out] output Output buffer. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_sha3_256(uint8_t *output, const uint8_t *input, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, SHA3_256_HASHBYTES)) + assigns(memory_slice(output, SHA3_256_HASHBYTES)) +); + +/* One-stop SHA3_512 call. Aliasing between input and + * output is not permitted */ +#define SHA3_512_HASHBYTES 64 +#define mlk_sha3_512 MLK_NAMESPACE(sha3_512) +/** + * SHA3-512 with non-incremental API. + * + * @param[out] output Output buffer. + * @param[in] input Input buffer. + * @param inlen Length of input in bytes. + */ +void mlk_sha3_512(uint8_t *output, const uint8_t *input, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(input, inlen)) + requires(memory_no_alias(output, SHA3_512_HASHBYTES)) + assigns(memory_slice(output, SHA3_512_HASHBYTES)) +); + +#if !defined(MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202) || \ + !defined(MLK_USE_NATIVE_FIPS202_X4) +/* If you provide your own FIPS-202 implementation where the x4- + * Keccak-f1600-x4 implementation falls back to 4-fold Keccak-f1600, + * set this to gain a small speedup. */ +#define FIPS202_X4_DEFAULT_IMPLEMENTATION +#endif /* !MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 || !MLK_USE_NATIVE_FIPS202_X4 \ + */ + + +#endif /* !MLK_FIPS202_FIPS202_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202x4.c b/src/mlkem-native/mlkem/src/fips202/fips202x4.c new file mode 100644 index 0000000000..400ca07109 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202x4.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#include "../common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "../verify.h" +#include "fips202.h" +#include "fips202x4.h" +#include "keccakf1600.h" + +typedef mlk_shake128x4ctx mlk_shake256x4_ctx; + +static void mlk_keccak_absorb_once_x4(uint64_t *s, unsigned r, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, + size_t inlen, uint8_t p) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(r > 0) + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY))) +{ + while (inlen >= r) + __loop__( + assigns(inlen, in0, in1, in2, in3, memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + invariant(inlen <= loop_entry(inlen)) + invariant(in0 == loop_entry(in0) + (loop_entry(inlen) - inlen)) + invariant(in1 == loop_entry(in1) + (loop_entry(inlen) - inlen)) + invariant(in2 == loop_entry(in2) + (loop_entry(inlen) - inlen)) + invariant(in3 == loop_entry(in3) + (loop_entry(inlen) - inlen)) + decreases(inlen)) + { + mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, r); + mlk_keccakf1600x4_permute(s); + + in0 += r; + in1 += r; + in2 += r; + in3 += r; + inlen -= r; + } + + /* At this point, inlen < r, so the truncations to unsigned are safe below. */ + + if (inlen > 0) + { + mlk_keccakf1600x4_xor_bytes(s, in0, in1, in2, in3, 0, (unsigned int)inlen); + } + + if (inlen == r - 1) + { + p |= 128; + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1); + } + else + { + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, (unsigned int)inlen, 1); + p = 128; + mlk_keccakf1600x4_xor_bytes(s, &p, &p, &p, &p, r - 1, 1); + } +} + +static void mlk_keccak_squeezeblocks_x4(uint8_t *out0, uint8_t *out1, + uint8_t *out2, uint8_t *out3, + size_t nblocks, uint64_t *s, unsigned r) +__contract__( + requires(r <= sizeof(uint64_t) * MLK_KECCAK_LANES) + requires(r == SHAKE128_RATE || r == SHAKE256_RATE) + requires(nblocks <= (MLK_MAX_BUFFER_SIZE / SHAKE256_RATE)) + requires(memory_no_alias(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(out0, nblocks * r)) + requires(memory_no_alias(out1, nblocks * r)) + requires(memory_no_alias(out2, nblocks * r)) + requires(memory_no_alias(out3, nblocks * r)) + assigns(memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + assigns(memory_slice(out0, nblocks * r)) + assigns(memory_slice(out1, nblocks * r)) + assigns(memory_slice(out2, nblocks * r)) + assigns(memory_slice(out3, nblocks * r))) +{ + size_t current_offset = 0; + while (nblocks > 0) + __loop__( + assigns(nblocks, current_offset, + memory_slice(s, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY), + memory_slice(out0, nblocks * r), + memory_slice(out1, nblocks * r), + memory_slice(out2, nblocks * r), + memory_slice(out3, nblocks * r)) + invariant(nblocks <= loop_entry(nblocks)) + invariant(current_offset == (loop_entry(nblocks) - nblocks) * r) + decreases(nblocks)) + { + mlk_keccakf1600x4_permute(s); + mlk_keccakf1600x4_extract_bytes( + s, &out0[current_offset], &out1[current_offset], &out2[current_offset], + &out3[current_offset], 0, r); + current_offset += r; + nblocks--; + } +} + +void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + const uint8_t *in3, size_t inlen) +{ + mlk_memset(state, 0, sizeof(mlk_shake128x4ctx)); + mlk_keccak_absorb_once_x4(state->ctx, SHAKE128_RATE, in0, in1, in2, in3, + inlen, 0x1F); +} + +void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + uint8_t *out3, size_t nblocks, + mlk_shake128x4ctx *state) +{ + mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx, + SHAKE128_RATE); +} + +void mlk_shake128x4_init(mlk_shake128x4ctx *state) { (void)state; } +void mlk_shake128x4_release(mlk_shake128x4ctx *state) +{ + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(state, sizeof(mlk_shake128x4ctx)); +} + +static void mlk_shake256x4_absorb_once(mlk_shake256x4_ctx *state, + const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, + size_t inlen) +{ + mlk_memset(state, 0, sizeof(mlk_shake128x4ctx)); + mlk_keccak_absorb_once_x4(state->ctx, SHAKE256_RATE, in0, in1, in2, in3, + inlen, 0x1F); +} + +static void mlk_shake256x4_squeezeblocks(uint8_t *out0, uint8_t *out1, + uint8_t *out2, uint8_t *out3, + size_t nblocks, + mlk_shake256x4_ctx *state) +{ + mlk_keccak_squeezeblocks_x4(out0, out1, out2, out3, nblocks, state->ctx, + SHAKE256_RATE); +} + +void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3, + size_t outlen, const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, size_t inlen) +{ + mlk_shake256x4_ctx statex; + size_t nblocks = outlen / SHAKE256_RATE; + uint8_t tmp0[SHAKE256_RATE]; + uint8_t tmp1[SHAKE256_RATE]; + uint8_t tmp2[SHAKE256_RATE]; + uint8_t tmp3[SHAKE256_RATE]; + + mlk_shake256x4_absorb_once(&statex, in0, in1, in2, in3, inlen); + mlk_shake256x4_squeezeblocks(out0, out1, out2, out3, nblocks, &statex); + + out0 += nblocks * SHAKE256_RATE; + out1 += nblocks * SHAKE256_RATE; + out2 += nblocks * SHAKE256_RATE; + out3 += nblocks * SHAKE256_RATE; + + outlen -= nblocks * SHAKE256_RATE; + + if (outlen) + { + mlk_shake256x4_squeezeblocks(tmp0, tmp1, tmp2, tmp3, 1, &statex); + mlk_memcpy(out0, tmp0, outlen); + mlk_memcpy(out1, tmp1, outlen); + mlk_memcpy(out2, tmp2, outlen); + mlk_memcpy(out3, tmp3, outlen); + } + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(&statex, sizeof(statex)); + mlk_zeroize(tmp0, sizeof(tmp0)); + mlk_zeroize(tmp1, sizeof(tmp1)); + mlk_zeroize(tmp2, sizeof(tmp2)); + mlk_zeroize(tmp3, sizeof(tmp3)); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202x4) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/fips202/fips202x4.h b/src/mlkem-native/mlkem/src/fips202/fips202x4.h new file mode 100644 index 0000000000..a0490a0974 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/fips202x4.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_FIPS202X4_H +#define MLK_FIPS202_FIPS202X4_H + + +#include "../cbmc.h" +#include "../common.h" + +#include "fips202.h" +#include "keccakf1600.h" + +/** Context for the non-incremental 4-way SHAKE128 API. */ +typedef struct +{ + uint64_t ctx[MLK_KECCAK_LANES * + MLK_KECCAK_WAY]; /**< 4-way Keccak state, stored sequentially. */ +} MLK_ALIGN mlk_shake128x4ctx; + +#define mlk_shake128x4_absorb_once MLK_NAMESPACE(shake128x4_absorb_once) +void mlk_shake128x4_absorb_once(mlk_shake128x4ctx *state, const uint8_t *in0, + const uint8_t *in1, const uint8_t *in2, + const uint8_t *in3, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx))) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + assigns(memory_slice(state, sizeof(mlk_shake128x4ctx))) +); + +#define mlk_shake128x4_squeezeblocks MLK_NAMESPACE(shake128x4_squeezeblocks) +void mlk_shake128x4_squeezeblocks(uint8_t *out0, uint8_t *out1, uint8_t *out2, + uint8_t *out3, size_t nblocks, + mlk_shake128x4ctx *state) +__contract__( + requires(nblocks <= 8 /* somewhat arbitrary bound */) + requires(memory_no_alias(state, sizeof(mlk_shake128x4ctx))) + requires(memory_no_alias(out0, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out1, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out2, nblocks * SHAKE128_RATE)) + requires(memory_no_alias(out3, nblocks * SHAKE128_RATE)) + assigns(memory_slice(out0, nblocks * SHAKE128_RATE), + memory_slice(out1, nblocks * SHAKE128_RATE), + memory_slice(out2, nblocks * SHAKE128_RATE), + memory_slice(out3, nblocks * SHAKE128_RATE), + memory_slice(state, sizeof(mlk_shake128x4ctx))) +); + +#define mlk_shake128x4_init MLK_NAMESPACE(shake128x4_init) +void mlk_shake128x4_init(mlk_shake128x4ctx *state); + +#define mlk_shake128x4_release MLK_NAMESPACE(shake128x4_release) +void mlk_shake128x4_release(mlk_shake128x4ctx *state); + +#define mlk_shake256x4 MLK_NAMESPACE(shake256x4) +void mlk_shake256x4(uint8_t *out0, uint8_t *out1, uint8_t *out2, uint8_t *out3, + size_t outlen, const uint8_t *in0, const uint8_t *in1, + const uint8_t *in2, const uint8_t *in3, size_t inlen) +__contract__( + requires(inlen <= MLK_MAX_BUFFER_SIZE) + requires(outlen <= MLK_MAX_BUFFER_SIZE) + requires(memory_no_alias(in0, inlen)) + requires(memory_no_alias(in1, inlen)) + requires(memory_no_alias(in2, inlen)) + requires(memory_no_alias(in3, inlen)) + requires(memory_no_alias(out0, outlen)) + requires(memory_no_alias(out1, outlen)) + requires(memory_no_alias(out2, outlen)) + requires(memory_no_alias(out3, outlen)) + assigns(memory_slice(out0, outlen)) + assigns(memory_slice(out1, outlen)) + assigns(memory_slice(out2, outlen)) + assigns(memory_slice(out3, outlen)) +); + +#endif /* !MLK_FIPS202_FIPS202X4_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/keccakf1600.c b/src/mlkem-native/mlkem/src/fips202/keccakf1600.c new file mode 100644 index 0000000000..b3cec2eaba --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/keccakf1600.c @@ -0,0 +1,499 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [mupq] + * Common files for pqm4, pqm3, pqriscv + * Kannwischer, Petri, Rijneveld, Schwabe, Stoffelen + * https://github.com/mupq/mupq + * + * - [supercop] + * SUPERCOP benchmarking framework + * Daniel J. Bernstein + * http://bench.cr.yp.to/supercop.html + * + * - [tweetfips] + * 'tweetfips202' FIPS202 implementation + * Van Assche, Bernstein, Schwabe + * https://keccak.team/2015/tweetfips202.html + */ + +/* Based on the CC0 implementation from @[mupq] and the public domain + * implementation @[supercop, crypto_hash/keccakc512/simple/] + * by Ronny Van Keer, and the public domain @[tweetfips] implementation. */ + + +#include "keccakf1600.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#define MLK_KECCAK_NROUNDS 24 +#define MLK_KECCAK_ROL(a, offset) (((a) << (offset)) ^ ((a) >> (64 - (offset)))) + +void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data, + unsigned offset, unsigned length) +{ + unsigned i; +#if defined(MLK_SYS_LITTLE_ENDIAN) + uint8_t *state_ptr = (uint8_t *)state + offset; + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + data[i] = state_ptr[i]; + } +#else /* MLK_SYS_LITTLE_ENDIAN */ + /* Portable version */ + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + data[i] = (state[(offset + i) >> 3] >> (8 * ((offset + i) & 0x07))) & 0xFF; + } +#endif /* !MLK_SYS_LITTLE_ENDIAN */ +} + +void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, + unsigned offset, unsigned length) +{ + unsigned i; +#if defined(MLK_SYS_LITTLE_ENDIAN) + uint8_t *state_ptr = (uint8_t *)state + offset; + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + state_ptr[i] ^= data[i]; + } +#else /* MLK_SYS_LITTLE_ENDIAN */ + /* Portable version */ + for (i = 0; i < length; i++) + __loop__(invariant(i <= length) + decreases(length - i)) + { + state[(offset + i) >> 3] ^= (uint64_t)data[i] + << (8 * ((offset + i) & 0x07)); + } +#endif /* !MLK_SYS_LITTLE_ENDIAN */ +} + +static void mlk_keccakf1600x4_extract_bytes_c(uint64_t *state, + unsigned char *data0, + unsigned char *data1, + unsigned char *data2, + unsigned char *data3, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) +) +{ + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 0, data0, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 1, data1, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 2, data2, offset, + length); + mlk_keccakf1600_extract_bytes(state + MLK_KECCAK_LANES * 3, data3, offset, + length); +} + +void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned char *data3, unsigned offset, + unsigned length) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES) + if (mlk_keccakf1600_extract_bytes_x4_native(state, data0, data1, data2, data3, + offset, length) == + MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES */ + mlk_keccakf1600x4_extract_bytes_c(state, data0, data1, data2, data3, offset, + length); +} + +static void mlk_keccakf1600x4_xor_bytes_c(uint64_t *state, + const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +) +{ + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 0, data0, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 1, data1, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 2, data2, offset, + length); + mlk_keccakf1600_xor_bytes(state + MLK_KECCAK_LANES * 3, data3, offset, + length); +} + +void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, unsigned offset, + unsigned length) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES) + if (mlk_keccakf1600_xor_bytes_x4_native(state, data0, data1, data2, data3, + offset, + length) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES */ + mlk_keccakf1600x4_xor_bytes_c(state, data0, data1, data2, data3, offset, + length); +} + +void mlk_keccakf1600x4_permute(uint64_t *state) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X4) + if (mlk_keccak_f1600_x4_native(state) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X4 */ + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 0); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 1); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 2); + mlk_keccakf1600_permute(state + MLK_KECCAK_LANES * 3); +} + +static const uint64_t mlk_KeccakF_RoundConstants[MLK_KECCAK_NROUNDS] = { + (uint64_t)0x0000000000000001ULL, (uint64_t)0x0000000000008082ULL, + (uint64_t)0x800000000000808aULL, (uint64_t)0x8000000080008000ULL, + (uint64_t)0x000000000000808bULL, (uint64_t)0x0000000080000001ULL, + (uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008009ULL, + (uint64_t)0x000000000000008aULL, (uint64_t)0x0000000000000088ULL, + (uint64_t)0x0000000080008009ULL, (uint64_t)0x000000008000000aULL, + (uint64_t)0x000000008000808bULL, (uint64_t)0x800000000000008bULL, + (uint64_t)0x8000000000008089ULL, (uint64_t)0x8000000000008003ULL, + (uint64_t)0x8000000000008002ULL, (uint64_t)0x8000000000000080ULL, + (uint64_t)0x000000000000800aULL, (uint64_t)0x800000008000000aULL, + (uint64_t)0x8000000080008081ULL, (uint64_t)0x8000000000008080ULL, + (uint64_t)0x0000000080000001ULL, (uint64_t)0x8000000080008008ULL}; + +MLK_STATIC_TESTABLE +void mlk_keccakf1600_permute_c(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +) +{ + unsigned round; + + uint64_t Aba, Abe, Abi, Abo, Abu; + uint64_t Aga, Age, Agi, Ago, Agu; + uint64_t Aka, Ake, Aki, Ako, Aku; + uint64_t Ama, Ame, Ami, Amo, Amu; + uint64_t Asa, Ase, Asi, Aso, Asu; + uint64_t BCa, BCe, BCi, BCo, BCu; + uint64_t Da, De, Di, Do, Du; + uint64_t Eba, Ebe, Ebi, Ebo, Ebu; + uint64_t Ega, Ege, Egi, Ego, Egu; + uint64_t Eka, Eke, Eki, Eko, Eku; + uint64_t Ema, Eme, Emi, Emo, Emu; + uint64_t Esa, Ese, Esi, Eso, Esu; + + /* copyFromState(A, state) */ + Aba = state[0]; + Abe = state[1]; + Abi = state[2]; + Abo = state[3]; + Abu = state[4]; + Aga = state[5]; + Age = state[6]; + Agi = state[7]; + Ago = state[8]; + Agu = state[9]; + Aka = state[10]; + Ake = state[11]; + Aki = state[12]; + Ako = state[13]; + Aku = state[14]; + Ama = state[15]; + Ame = state[16]; + Ami = state[17]; + Amo = state[18]; + Amu = state[19]; + Asa = state[20]; + Ase = state[21]; + Asi = state[22]; + Aso = state[23]; + Asu = state[24]; + + for (round = 0; round < MLK_KECCAK_NROUNDS; round += 2) + __loop__(invariant(round <= MLK_KECCAK_NROUNDS && round % 2 == 0) + decreases(MLK_KECCAK_NROUNDS - round)) + { + /* prepareTheta */ + BCa = Aba ^ Aga ^ Aka ^ Ama ^ Asa; + BCe = Abe ^ Age ^ Ake ^ Ame ^ Ase; + BCi = Abi ^ Agi ^ Aki ^ Ami ^ Asi; + BCo = Abo ^ Ago ^ Ako ^ Amo ^ Aso; + BCu = Abu ^ Agu ^ Aku ^ Amu ^ Asu; + + /* thetaRhoPiChiIotaPrepareTheta(round, A, E) */ + Da = BCu ^ MLK_KECCAK_ROL(BCe, 1); + De = BCa ^ MLK_KECCAK_ROL(BCi, 1); + Di = BCe ^ MLK_KECCAK_ROL(BCo, 1); + Do = BCi ^ MLK_KECCAK_ROL(BCu, 1); + Du = BCo ^ MLK_KECCAK_ROL(BCa, 1); + + Aba ^= Da; + BCa = Aba; + Age ^= De; + BCe = MLK_KECCAK_ROL(Age, 44); + Aki ^= Di; + BCi = MLK_KECCAK_ROL(Aki, 43); + Amo ^= Do; + BCo = MLK_KECCAK_ROL(Amo, 21); + Asu ^= Du; + BCu = MLK_KECCAK_ROL(Asu, 14); + Eba = BCa ^ ((~BCe) & BCi); + Eba ^= (uint64_t)mlk_KeccakF_RoundConstants[round]; + Ebe = BCe ^ ((~BCi) & BCo); + Ebi = BCi ^ ((~BCo) & BCu); + Ebo = BCo ^ ((~BCu) & BCa); + Ebu = BCu ^ ((~BCa) & BCe); + + Abo ^= Do; + BCa = MLK_KECCAK_ROL(Abo, 28); + Agu ^= Du; + BCe = MLK_KECCAK_ROL(Agu, 20); + Aka ^= Da; + BCi = MLK_KECCAK_ROL(Aka, 3); + Ame ^= De; + BCo = MLK_KECCAK_ROL(Ame, 45); + Asi ^= Di; + BCu = MLK_KECCAK_ROL(Asi, 61); + Ega = BCa ^ ((~BCe) & BCi); + Ege = BCe ^ ((~BCi) & BCo); + Egi = BCi ^ ((~BCo) & BCu); + Ego = BCo ^ ((~BCu) & BCa); + Egu = BCu ^ ((~BCa) & BCe); + + Abe ^= De; + BCa = MLK_KECCAK_ROL(Abe, 1); + Agi ^= Di; + BCe = MLK_KECCAK_ROL(Agi, 6); + Ako ^= Do; + BCi = MLK_KECCAK_ROL(Ako, 25); + Amu ^= Du; + BCo = MLK_KECCAK_ROL(Amu, 8); + Asa ^= Da; + BCu = MLK_KECCAK_ROL(Asa, 18); + Eka = BCa ^ ((~BCe) & BCi); + Eke = BCe ^ ((~BCi) & BCo); + Eki = BCi ^ ((~BCo) & BCu); + Eko = BCo ^ ((~BCu) & BCa); + Eku = BCu ^ ((~BCa) & BCe); + + Abu ^= Du; + BCa = MLK_KECCAK_ROL(Abu, 27); + Aga ^= Da; + BCe = MLK_KECCAK_ROL(Aga, 36); + Ake ^= De; + BCi = MLK_KECCAK_ROL(Ake, 10); + Ami ^= Di; + BCo = MLK_KECCAK_ROL(Ami, 15); + Aso ^= Do; + BCu = MLK_KECCAK_ROL(Aso, 56); + Ema = BCa ^ ((~BCe) & BCi); + Eme = BCe ^ ((~BCi) & BCo); + Emi = BCi ^ ((~BCo) & BCu); + Emo = BCo ^ ((~BCu) & BCa); + Emu = BCu ^ ((~BCa) & BCe); + + Abi ^= Di; + BCa = MLK_KECCAK_ROL(Abi, 62); + Ago ^= Do; + BCe = MLK_KECCAK_ROL(Ago, 55); + Aku ^= Du; + BCi = MLK_KECCAK_ROL(Aku, 39); + Ama ^= Da; + BCo = MLK_KECCAK_ROL(Ama, 41); + Ase ^= De; + BCu = MLK_KECCAK_ROL(Ase, 2); + Esa = BCa ^ ((~BCe) & BCi); + Ese = BCe ^ ((~BCi) & BCo); + Esi = BCi ^ ((~BCo) & BCu); + Eso = BCo ^ ((~BCu) & BCa); + Esu = BCu ^ ((~BCa) & BCe); + + /* prepareTheta */ + BCa = Eba ^ Ega ^ Eka ^ Ema ^ Esa; + BCe = Ebe ^ Ege ^ Eke ^ Eme ^ Ese; + BCi = Ebi ^ Egi ^ Eki ^ Emi ^ Esi; + BCo = Ebo ^ Ego ^ Eko ^ Emo ^ Eso; + BCu = Ebu ^ Egu ^ Eku ^ Emu ^ Esu; + + /* thetaRhoPiChiIotaPrepareTheta(round+1, E, A) */ + Da = BCu ^ MLK_KECCAK_ROL(BCe, 1); + De = BCa ^ MLK_KECCAK_ROL(BCi, 1); + Di = BCe ^ MLK_KECCAK_ROL(BCo, 1); + Do = BCi ^ MLK_KECCAK_ROL(BCu, 1); + Du = BCo ^ MLK_KECCAK_ROL(BCa, 1); + + Eba ^= Da; + BCa = Eba; + Ege ^= De; + BCe = MLK_KECCAK_ROL(Ege, 44); + Eki ^= Di; + BCi = MLK_KECCAK_ROL(Eki, 43); + Emo ^= Do; + BCo = MLK_KECCAK_ROL(Emo, 21); + Esu ^= Du; + BCu = MLK_KECCAK_ROL(Esu, 14); + Aba = BCa ^ ((~BCe) & BCi); + Aba ^= (uint64_t)mlk_KeccakF_RoundConstants[round + 1]; + Abe = BCe ^ ((~BCi) & BCo); + Abi = BCi ^ ((~BCo) & BCu); + Abo = BCo ^ ((~BCu) & BCa); + Abu = BCu ^ ((~BCa) & BCe); + + Ebo ^= Do; + BCa = MLK_KECCAK_ROL(Ebo, 28); + Egu ^= Du; + BCe = MLK_KECCAK_ROL(Egu, 20); + Eka ^= Da; + BCi = MLK_KECCAK_ROL(Eka, 3); + Eme ^= De; + BCo = MLK_KECCAK_ROL(Eme, 45); + Esi ^= Di; + BCu = MLK_KECCAK_ROL(Esi, 61); + Aga = BCa ^ ((~BCe) & BCi); + Age = BCe ^ ((~BCi) & BCo); + Agi = BCi ^ ((~BCo) & BCu); + Ago = BCo ^ ((~BCu) & BCa); + Agu = BCu ^ ((~BCa) & BCe); + + Ebe ^= De; + BCa = MLK_KECCAK_ROL(Ebe, 1); + Egi ^= Di; + BCe = MLK_KECCAK_ROL(Egi, 6); + Eko ^= Do; + BCi = MLK_KECCAK_ROL(Eko, 25); + Emu ^= Du; + BCo = MLK_KECCAK_ROL(Emu, 8); + Esa ^= Da; + BCu = MLK_KECCAK_ROL(Esa, 18); + Aka = BCa ^ ((~BCe) & BCi); + Ake = BCe ^ ((~BCi) & BCo); + Aki = BCi ^ ((~BCo) & BCu); + Ako = BCo ^ ((~BCu) & BCa); + Aku = BCu ^ ((~BCa) & BCe); + + Ebu ^= Du; + BCa = MLK_KECCAK_ROL(Ebu, 27); + Ega ^= Da; + BCe = MLK_KECCAK_ROL(Ega, 36); + Eke ^= De; + BCi = MLK_KECCAK_ROL(Eke, 10); + Emi ^= Di; + BCo = MLK_KECCAK_ROL(Emi, 15); + Eso ^= Do; + BCu = MLK_KECCAK_ROL(Eso, 56); + Ama = BCa ^ ((~BCe) & BCi); + Ame = BCe ^ ((~BCi) & BCo); + Ami = BCi ^ ((~BCo) & BCu); + Amo = BCo ^ ((~BCu) & BCa); + Amu = BCu ^ ((~BCa) & BCe); + + Ebi ^= Di; + BCa = MLK_KECCAK_ROL(Ebi, 62); + Ego ^= Do; + BCe = MLK_KECCAK_ROL(Ego, 55); + Eku ^= Du; + BCi = MLK_KECCAK_ROL(Eku, 39); + Ema ^= Da; + BCo = MLK_KECCAK_ROL(Ema, 41); + Ese ^= De; + BCu = MLK_KECCAK_ROL(Ese, 2); + Asa = BCa ^ ((~BCe) & BCi); + Ase = BCe ^ ((~BCi) & BCo); + Asi = BCi ^ ((~BCo) & BCu); + Aso = BCo ^ ((~BCu) & BCa); + Asu = BCu ^ ((~BCa) & BCe); + } + + /* copyToState(state, A) */ + state[0] = Aba; + state[1] = Abe; + state[2] = Abi; + state[3] = Abo; + state[4] = Abu; + state[5] = Aga; + state[6] = Age; + state[7] = Agi; + state[8] = Ago; + state[9] = Agu; + state[10] = Aka; + state[11] = Ake; + state[12] = Aki; + state[13] = Ako; + state[14] = Aku; + state[15] = Ama; + state[16] = Ame; + state[17] = Ami; + state[18] = Amo; + state[19] = Amu; + state[20] = Asa; + state[21] = Ase; + state[22] = Asi; + state[23] = Aso; + state[24] = Asu; +} + +void mlk_keccakf1600_permute(uint64_t *state) +{ +#if defined(MLK_USE_NATIVE_FIPS202_X1) + if (mlk_keccak_f1600_x1_native(state) == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_FIPS202_X1 */ + mlk_keccakf1600_permute_c(state); +} + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(keccakf1600) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_KECCAK_NROUNDS +#undef MLK_KECCAK_ROL diff --git a/src/mlkem-native/mlkem/src/fips202/keccakf1600.h b/src/mlkem-native/mlkem/src/fips202/keccakf1600.h new file mode 100644 index 0000000000..c26c36a084 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/keccakf1600.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_KECCAKF1600_H +#define MLK_FIPS202_KECCAKF1600_H +#include "../cbmc.h" +#include "../common.h" + +#define MLK_KECCAK_LANES 25 +#define MLK_KECCAK_WAY 4 + +/* + * WARNING: + * The contents of this structure, including the placement + * and interleaving of Keccak lanes, are IMPLEMENTATION-DEFINED. + * The struct is only exposed here to allow its construction on the stack. + */ + +#define mlk_keccakf1600_extract_bytes MLK_NAMESPACE(keccakf1600_extract_bytes) +void mlk_keccakf1600_extract_bytes(uint64_t *state, unsigned char *data, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(data, length)) + assigns(memory_slice(data, length)) +); + +#define mlk_keccakf1600_xor_bytes MLK_NAMESPACE(keccakf1600_xor_bytes) +void mlk_keccakf1600_xor_bytes(uint64_t *state, const unsigned char *data, + unsigned offset, unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + requires(memory_no_alias(data, length)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +); + +#define mlk_keccakf1600x4_extract_bytes \ + MLK_NAMESPACE(keccakf1600x4_extract_bytes) +void mlk_keccakf1600x4_extract_bytes(uint64_t *state, unsigned char *data0, + unsigned char *data1, unsigned char *data2, + unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) +); + +#define mlk_keccakf1600x4_xor_bytes MLK_NAMESPACE(keccakf1600x4_xor_bytes) +void mlk_keccakf1600x4_xor_bytes(uint64_t *state, const unsigned char *data0, + const unsigned char *data1, + const unsigned char *data2, + const unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= MLK_KECCAK_LANES * sizeof(uint64_t) && + 0 <= length && length <= MLK_KECCAK_LANES * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + requires(memory_no_alias(data0, length)) + /* Case 1: all input buffers are distinct; Case 2: All input buffers are the same */ + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +); + + +#define mlk_keccakf1600x4_permute MLK_NAMESPACE(keccakf1600x4_permute) +void mlk_keccakf1600x4_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES * MLK_KECCAK_WAY)) +); + +#define mlk_keccakf1600_permute MLK_NAMESPACE(keccakf1600_permute) +void mlk_keccakf1600_permute(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) + assigns(memory_slice(state, sizeof(uint64_t) * MLK_KECCAK_LANES)) +); + +#endif /* !MLK_FIPS202_KECCAKF1600_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h new file mode 100644 index 0000000000..b22bfe41cb --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/auto.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_AUTO_H +#define MLK_FIPS202_NATIVE_AARCH64_AUTO_H +/* Default FIPS202 assembly profile for AArch64 systems */ + +/* + * Default logic to decide which implementation to use. + * + */ + +/* + * Keccak-f1600 + * + * - On Arm-based Apple CPUs, or if MLK_SYS_AARCH64_FAST_SHA3 is set, + * we pick a pure Neon implementation. + * - Otherwise, unless MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set, + * we use lazy-rotation scalar assembly from @[HYBRID]. + * - Otherwise, if MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER is set, we + * fall back to the standard C implementation. + */ +#if defined(__ARM_FEATURE_SHA3) && \ + (defined(__APPLE__) || defined(MLK_SYS_AARCH64_FAST_SHA3)) +#include "x1_v84a.h" +#elif !defined(MLK_SYS_AARCH64_SLOW_BARREL_SHIFTER) +#include "x1_scalar.h" +#endif + +/* Batched, SIMD-based Keccak-f1600 implementations. */ +#if defined(MLK_SYS_AARCH64_NEON) + +/* + * Keccak-f1600x2/x4 + * + * The optimal implementation is highly CPU-specific; see @[HYBRID]. + * + * For now, if v8.4-A is not implemented, we fall back to Keccak-f1600. + * If v8.4-A is implemented and we are on an Apple CPU or + * MLK_SYS_AARCH64_FAST_SHA3 is set, we use a plain Neon-based + * implementation. + * Otherwise, if v8.4-A is implemented, we use a scalar/Neon/Neon hybrid. + * The reason for this distinction is that Apple CPUs (and CPUs flagged with + * MLK_SYS_AARCH64_FAST_SHA3) implement the SHA3 instructions on all SIMD + * units, while Arm CPUs prior to Cortex-X4 don't, and ordinary Neon + * instructions are still needed. + */ +#if defined(__ARM_FEATURE_SHA3) +/* + * For Apple-M cores (and CPUs flagged with MLK_SYS_AARCH64_FAST_SHA3), we + * use a plain implementation leveraging SHA3 instructions only. + */ +#if defined(__APPLE__) || defined(MLK_SYS_AARCH64_FAST_SHA3) +#include "x2_v84a.h" +#else +#include "x4_v8a_v84a_scalar.h" +#endif + +#else /* __ARM_FEATURE_SHA3 */ + +#include "x4_v8a_scalar.h" + +#endif /* !__ARM_FEATURE_SHA3 */ + +#endif /* MLK_SYS_AARCH64_NEON */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_AUTO_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h new file mode 100644 index 0000000000..08b105b212 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/fips202_native_aarch64.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H +#define MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint64_t + mlk_keccakf1600_round_constants[24]; + +#define mlk_keccak_f1600_x1_scalar_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) +void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_scalar_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#define mlk_keccak_f1600_x1_v84a_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm) +void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x1_v84a_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) +); + +#define mlk_keccak_f1600_x2_v84a_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm) +void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/keccak_f1600_x2_v84a_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 2)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 2)) +); + +#define mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) +void mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], + const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in + * proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) +); + +#define mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm \ + MLK_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) +void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm( + uint64_t state[100], const uint64_t rc[24]) +/* This must be kept in sync with the HOL-Light specification + * in + * proofs/hol_light/aarch64/proofs/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) +); + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_SRC_FIPS202_NATIVE_AARCH64_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S new file mode 100644 index 0000000000..108e17f83f --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S @@ -0,0 +1,377 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x1_scalar_aarch64_asm + Description: AArch64 scalar implementation of Keccak-f[1600] permutation for single state + Signature: void mlk_keccak_f1600_x1_scalar_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: uint64_t const *rc + description: Round constants (24 x uint64_t) + Stack: + bytes: 128 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x1_scalar_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x1_scalar_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x1_scalar_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x80 + .cfi_adjust_cfa_offset 0x80 + stp x19, x20, [sp, #0x20] + .cfi_rel_offset x19, 0x20 + .cfi_rel_offset x20, 0x28 + stp x21, x22, [sp, #0x30] + .cfi_rel_offset x21, 0x30 + .cfi_rel_offset x22, 0x38 + stp x23, x24, [sp, #0x40] + .cfi_rel_offset x23, 0x40 + .cfi_rel_offset x24, 0x48 + stp x25, x26, [sp, #0x50] + .cfi_rel_offset x25, 0x50 + .cfi_rel_offset x26, 0x58 + stp x27, x28, [sp, #0x60] + .cfi_rel_offset x27, 0x60 + .cfi_rel_offset x28, 0x68 + stp x29, x30, [sp, #0x70] + .cfi_rel_offset x29, 0x70 + .cfi_rel_offset x30, 0x78 + +Lmlk_keccak_f1600_x1_scalar_initial: + mov x26, x1 + str x1, [sp, #0x8] + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + str x0, [sp] + eor x30, x24, x25 + eor x27, x9, x10 + eor x0, x30, x21 + eor x26, x27, x6 + eor x27, x26, x7 + eor x29, x0, x22 + eor x26, x29, x23 + eor x29, x4, x5 + eor x30, x29, x1 + eor x0, x27, x8 + eor x29, x30, x2 + eor x30, x19, x20 + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor x4, x4, x27 + eor x30, x30, x17 + eor x30, x30, x28 + eor x29, x29, x3 + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor x23, x23, x30 + str x23, [sp, #0x18] + eor x23, x14, x15 + eor x14, x14, x0 + eor x23, x23, x11 + eor x15, x15, x0 + eor x1, x1, x27 + eor x23, x23, x12 + eor x23, x23, x13 + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + eor x26, x13, x0 + eor x13, x28, x23 + eor x28, x24, x30 + eor x24, x16, x23 + eor x16, x21, x30 + eor x21, x25, x30 + eor x30, x19, x23 + eor x19, x20, x23 + eor x20, x17, x23 + eor x17, x12, x0 + eor x0, x2, x27 + eor x2, x6, x29 + eor x6, x8, x29 + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + ldr x27, [sp, #0x18] + bic x25, x17, x2, ror #5 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + bic x7, x5, x28, ror #10 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor x5, x20, x11, ror #41 + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + eor x17, x24, x9, ror #47 + mov x24, #0x1 // =1 + bic x9, x0, x16, ror #9 + str x24, [sp, #0x10] + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + ldr x11, [x11] + bic x4, x21, x30, ror #57 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 + +Lmlk_keccak_f1600_x1_scalar_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + str x28, [sp, #0x18] + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x10] + eor x17, x10, x9, ror #47 + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0x18] + add x25, x25, #0x1 + str x25, [sp, #0x10] + cmp x25, #0x17 + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + b.le Lmlk_keccak_f1600_x1_scalar_loop + ror x6, x6, #0x2b + ror x11, x11, #0x32 + ror x21, x21, #0x14 + ror x2, x2, #0x3d + ror x7, x7, #0x13 + ror x12, x12, #0x3 + ror x17, x17, #0x24 + ror x22, x22, #0x2c + ror x3, x3, #0x27 + ror x8, x8, #0x38 + ror x13, x13, #0x2e + ror x28, x28, #0x3f + ror x23, x23, #0x3a + ror x4, x4, #0x36 + ror x9, x9, #0x31 + ror x14, x14, #0x8 + ror x19, x19, #0x25 + ror x24, x24, #0x1c + ror x5, x5, #0x19 + ror x10, x10, #0x17 + ror x15, x15, #0x3e + ror x20, x20, #0x2 + ror x25, x25, #0x9 + ldr x0, [sp] + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + ldp x19, x20, [sp, #0x20] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x30] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x40] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x50] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x60] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x70] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0x80 + .cfi_adjust_cfa_offset -0x80 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x1_scalar_aarch64_asm) + +#endif /* MLK_FIPS202_AARCH64_NEED_X1_SCALAR && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S new file mode 100644 index 0000000000..cb27677691 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S @@ -0,0 +1,206 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +/*yaml + Name: keccak_f1600_x1_v84a_aarch64_asm + Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for single state + Signature: void mlk_keccak_f1600_x1_v84a_aarch64_asm(uint64_t state[25], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 200 + permissions: read/write + c_parameter: uint64_t state[25] + description: Keccak state (25 x uint64_t) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 64 + description: register preservation +*/ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// +// This implementation is essentially from the paper @[HYBRID]. +// The only difference is interleaving/deinterleaving of Keccak state +// during load and store, so that the caller need not do this. +// + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x1_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x1_v84a_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x1_v84a_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldp d0, d1, [x0] + ldp d2, d3, [x0, #0x10] + ldp d4, d5, [x0, #0x20] + ldp d6, d7, [x0, #0x30] + ldp d8, d9, [x0, #0x40] + ldp d10, d11, [x0, #0x50] + ldp d12, d13, [x0, #0x60] + ldp d14, d15, [x0, #0x70] + ldp d16, d17, [x0, #0x80] + ldp d18, d19, [x0, #0x90] + ldp d20, d21, [x0, #0xa0] + ldp d22, d23, [x0, #0xb0] + ldr d24, [x0, #0xc0] + mov x2, #0x18 // =24 + +Lmlk_keccak_f1600_x1_v84a_loop: + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor3 v26.16b, v26.16b, v19.16b, v24.16b + rax1 v25.2d, v30.2d, v28.2d + rax1 v28.2d, v28.2d, v26.2d + rax1 v26.2d, v26.2d, v29.2d + rax1 v29.2d, v29.2d, v27.2d + rax1 v27.2d, v27.2d, v30.2d + eor v30.16b, v0.16b, v26.16b + xar v0.2d, v2.2d, v29.2d, #0x2 + xar v2.2d, v12.2d, v29.2d, #0x15 + xar v12.2d, v13.2d, v28.2d, #0x27 + xar v13.2d, v19.2d, v27.2d, #0x38 + xar v19.2d, v23.2d, v28.2d, #0x8 + xar v23.2d, v15.2d, v26.2d, #0x17 + xar v15.2d, v1.2d, v25.2d, #0x3f + xar v1.2d, v8.2d, v28.2d, #0x9 + xar v8.2d, v16.2d, v25.2d, #0x13 + xar v16.2d, v7.2d, v29.2d, #0x3a + xar v7.2d, v10.2d, v26.2d, #0x3d + xar v10.2d, v3.2d, v28.2d, #0x24 + xar v3.2d, v18.2d, v28.2d, #0x2b + xar v18.2d, v17.2d, v29.2d, #0x31 + xar v17.2d, v11.2d, v25.2d, #0x36 + xar v11.2d, v9.2d, v27.2d, #0x2c + xar v9.2d, v22.2d, v29.2d, #0x3 + xar v22.2d, v14.2d, v27.2d, #0x19 + xar v14.2d, v20.2d, v26.2d, #0x2e + xar v20.2d, v4.2d, v27.2d, #0x25 + xar v4.2d, v24.2d, v27.2d, #0x32 + xar v24.2d, v21.2d, v25.2d, #0x3e + xar v21.2d, v5.2d, v26.2d, #0x1c + xar v27.2d, v6.2d, v25.2d, #0x14 + ld1r { v31.2d }, [x1], #8 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bcax v6.16b, v11.16b, v8.16b, v7.16b + bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax v8.16b, v8.16b, v10.16b, v9.16b + bcax v9.16b, v9.16b, v11.16b, v10.16b + bcax v10.16b, v15.16b, v12.16b, v16.16b + bcax v11.16b, v16.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v15.16b, v14.16b + bcax v14.16b, v14.16b, v16.16b, v15.16b + bcax v15.16b, v20.16b, v17.16b, v21.16b + bcax v16.16b, v21.16b, v18.16b, v17.16b + bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax v18.16b, v18.16b, v20.16b, v19.16b + bcax v19.16b, v19.16b, v21.16b, v20.16b + bcax v20.16b, v0.16b, v22.16b, v1.16b + bcax v21.16b, v1.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v0.16b, v24.16b + bcax v24.16b, v24.16b, v1.16b, v0.16b + bcax v0.16b, v30.16b, v2.16b, v27.16b + bcax v1.16b, v27.16b, v3.16b, v2.16b + bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax v3.16b, v3.16b, v30.16b, v4.16b + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor v0.16b, v0.16b, v31.16b + sub x2, x2, #0x1 + cbnz x2, Lmlk_keccak_f1600_x1_v84a_loop + stp d0, d1, [x0] + stp d2, d3, [x0, #0x10] + stp d4, d5, [x0, #0x20] + stp d6, d7, [x0, #0x30] + stp d8, d9, [x0, #0x40] + stp d10, d11, [x0, #0x50] + stp d12, d13, [x0, #0x60] + stp d14, d15, [x0, #0x70] + stp d16, d17, [x0, #0x80] + stp d18, d19, [x0, #0x90] + stp d20, d21, [x0, #0xa0] + stp d22, d23, [x0, #0xb0] + str d24, [x0, #0xc0] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x1_v84a_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X1_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S new file mode 100644 index 0000000000..36ebe1d32f --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S @@ -0,0 +1,261 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [HYBRID] + * Hybrid scalar/vector implementations of Keccak and SPHINCS+ on AArch64 + * Becker, Kannwischer + * https://eprint.iacr.org/2022/1243 + */ + +/*yaml + Name: keccak_f1600_x2_v84a_aarch64_asm + Description: AArch64 ARMv8.4-A implementation of Keccak-f[1600] permutation for two sequential states + Signature: void mlk_keccak_f1600_x2_v84a_aarch64_asm(uint64_t state[50], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 400 + permissions: read/write + c_parameter: uint64_t state[50] + description: Two sequential Keccak states (state0[25], state1[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 64 + description: register preservation +*/ + +// +// Author: Hanno Becker +// Author: Matthias Kannwischer +// +// This implementation is essentially from the paper @[HYBRID]. +// The only difference is interleaving/deinterleaving of Keccak state +// during load and store, so that the caller need not do this. +// + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x2_v84a_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x2_v84a_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x2_v84a_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + add x2, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x2], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x2] + trn1 v24.2d, v25.2d, v27.2d + mov x2, #0x18 // =24 + +Lmlk_keccak_f1600_x2_v84a_loop: + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor3 v30.16b, v30.16b, v15.16b, v20.16b + eor3 v29.16b, v29.16b, v16.16b, v21.16b + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 v27.16b, v27.16b, v18.16b, v23.16b + eor3 v26.16b, v26.16b, v19.16b, v24.16b + rax1 v25.2d, v30.2d, v28.2d + rax1 v28.2d, v28.2d, v26.2d + rax1 v26.2d, v26.2d, v29.2d + rax1 v29.2d, v29.2d, v27.2d + rax1 v27.2d, v27.2d, v30.2d + eor v30.16b, v0.16b, v26.16b + xar v0.2d, v2.2d, v29.2d, #0x2 + xar v2.2d, v12.2d, v29.2d, #0x15 + xar v12.2d, v13.2d, v28.2d, #0x27 + xar v13.2d, v19.2d, v27.2d, #0x38 + xar v19.2d, v23.2d, v28.2d, #0x8 + xar v23.2d, v15.2d, v26.2d, #0x17 + xar v15.2d, v1.2d, v25.2d, #0x3f + xar v1.2d, v8.2d, v28.2d, #0x9 + xar v8.2d, v16.2d, v25.2d, #0x13 + xar v16.2d, v7.2d, v29.2d, #0x3a + xar v7.2d, v10.2d, v26.2d, #0x3d + xar v10.2d, v3.2d, v28.2d, #0x24 + xar v3.2d, v18.2d, v28.2d, #0x2b + xar v18.2d, v17.2d, v29.2d, #0x31 + xar v17.2d, v11.2d, v25.2d, #0x36 + xar v11.2d, v9.2d, v27.2d, #0x2c + xar v9.2d, v22.2d, v29.2d, #0x3 + xar v22.2d, v14.2d, v27.2d, #0x19 + xar v14.2d, v20.2d, v26.2d, #0x2e + xar v20.2d, v4.2d, v27.2d, #0x25 + xar v4.2d, v24.2d, v27.2d, #0x32 + xar v24.2d, v21.2d, v25.2d, #0x3e + xar v21.2d, v5.2d, v26.2d, #0x1c + xar v27.2d, v6.2d, v25.2d, #0x14 + ld1r { v31.2d }, [x1], #8 + bcax v5.16b, v10.16b, v7.16b, v11.16b + bcax v6.16b, v11.16b, v8.16b, v7.16b + bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax v8.16b, v8.16b, v10.16b, v9.16b + bcax v9.16b, v9.16b, v11.16b, v10.16b + bcax v10.16b, v15.16b, v12.16b, v16.16b + bcax v11.16b, v16.16b, v13.16b, v12.16b + bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax v13.16b, v13.16b, v15.16b, v14.16b + bcax v14.16b, v14.16b, v16.16b, v15.16b + bcax v15.16b, v20.16b, v17.16b, v21.16b + bcax v16.16b, v21.16b, v18.16b, v17.16b + bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax v18.16b, v18.16b, v20.16b, v19.16b + bcax v19.16b, v19.16b, v21.16b, v20.16b + bcax v20.16b, v0.16b, v22.16b, v1.16b + bcax v21.16b, v1.16b, v23.16b, v22.16b + bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax v23.16b, v23.16b, v0.16b, v24.16b + bcax v24.16b, v24.16b, v1.16b, v0.16b + bcax v0.16b, v30.16b, v2.16b, v27.16b + bcax v1.16b, v27.16b, v3.16b, v2.16b + bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax v3.16b, v3.16b, v30.16b, v4.16b + bcax v4.16b, v4.16b, v27.16b, v30.16b + eor v0.16b, v0.16b, v31.16b + sub x2, x2, #0x1 + cbnz x2, Lmlk_keccak_f1600_x2_v84a_loop + sub x0, x0, #0xc0 + add x2, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x2], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x2], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x2] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x2_v84a_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X2_V84A && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..7f0f78b340 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,1079 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for four sequential states + Signature: void mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t state[100] + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + mov x30, #0x0 // =0 + str x30, [sp, #0x20] + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor v30.16b, v0.16b, v5.16b + eor v30.16b, v30.16b, v10.16b + eor x0, x30, x21 + eor v30.16b, v30.16b, v15.16b + eor x26, x27, x6 + eor x27, x26, x7 + eor v30.16b, v30.16b, v20.16b + eor x29, x0, x22 + eor v29.16b, v1.16b, v6.16b + eor x26, x29, x23 + eor v29.16b, v29.16b, v11.16b + eor x29, x4, x5 + eor x30, x29, x1 + eor v29.16b, v29.16b, v16.16b + eor x0, x27, x8 + eor v29.16b, v29.16b, v21.16b + eor x29, x30, x2 + eor v28.16b, v2.16b, v7.16b + eor x30, x19, x20 + eor x30, x30, x16 + eor v28.16b, v28.16b, v12.16b + eor x27, x26, x0, ror #63 + eor v28.16b, v28.16b, v17.16b + eor x4, x4, x27 + eor v28.16b, v28.16b, v22.16b + eor x30, x30, x17 + eor x30, x30, x28 + eor v27.16b, v3.16b, v8.16b + eor x29, x29, x3 + eor v27.16b, v27.16b, v13.16b + eor x0, x0, x30, ror #63 + eor v27.16b, v27.16b, v18.16b + eor x30, x30, x29, ror #63 + eor x22, x22, x30 + eor v27.16b, v27.16b, v23.16b + eor x23, x23, x30 + eor v26.16b, v4.16b, v9.16b + str x23, [sp, #0xd0] + eor v26.16b, v26.16b, v14.16b + eor x23, x14, x15 + eor x14, x14, x0 + eor v26.16b, v26.16b, v19.16b + eor x23, x23, x11 + eor v26.16b, v26.16b, v24.16b + eor x15, x15, x0 + eor x1, x1, x27 + add v31.2d, v28.2d, v28.2d + eor x23, x23, x12 + sri v31.2d, v28.2d, #0x3f + eor x23, x23, x13 + eor v25.16b, v31.16b, v30.16b + eor x11, x11, x0 + eor x29, x29, x23, ror #63 + add v31.2d, v26.2d, v26.2d + eor x23, x23, x26, ror #63 + sri v31.2d, v26.2d, #0x3f + eor x26, x13, x0 + eor v28.16b, v31.16b, v28.16b + eor x13, x28, x23 + eor x28, x24, x30 + add v31.2d, v29.2d, v29.2d + eor x24, x16, x23 + sri v31.2d, v29.2d, #0x3f + eor x16, x21, x30 + eor v26.16b, v31.16b, v26.16b + eor x21, x25, x30 + eor x30, x19, x23 + add v31.2d, v27.2d, v27.2d + eor x19, x20, x23 + sri v31.2d, v27.2d, #0x3f + eor x20, x17, x23 + eor v29.16b, v31.16b, v29.16b + eor x17, x12, x0 + eor x0, x2, x27 + add v31.2d, v30.2d, v30.2d + eor x2, x6, x29 + sri v31.2d, v30.2d, #0x3f + eor x6, x8, x29 + eor v27.16b, v31.16b, v27.16b + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + eor v30.16b, v0.16b, v26.16b + bic x3, x13, x17, ror #19 + eor v31.16b, v2.16b, v29.16b + eor x5, x5, x27 + ldr x27, [sp, #0xd0] + shl v0.2d, v31.2d, #0x3e + bic x25, x17, x2, ror #5 + sri v0.2d, v31.2d, #0x2 + eor x9, x9, x29 + eor v31.16b, v12.16b, v29.16b + eor x23, x25, x5, ror #52 + eor x3, x3, x2, ror #24 + shl v2.2d, v31.2d, #0x2b + eor x8, x8, x17, ror #2 + sri v2.2d, v31.2d, #0x15 + eor x17, x10, x29 + eor v31.16b, v13.16b, v28.16b + bic x25, x12, x22, ror #47 + eor x29, x7, x29 + shl v12.2d, v31.2d, #0x19 + bic x10, x4, x27, ror #2 + sri v12.2d, v31.2d, #0x27 + bic x7, x5, x28, ror #10 + eor v31.16b, v19.16b, v27.16b + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + shl v13.2d, v31.2d, #0x8 + bic x7, x2, x5, ror #47 + sri v13.2d, v31.2d, #0x38 + eor x2, x25, x24, ror #39 + eor v31.16b, v23.16b, v28.16b + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + shl v19.2d, v31.2d, #0x38 + eor x25, x25, x17, ror #53 + sri v19.2d, v31.2d, #0x8 + bic x17, x11, x17, ror #60 + eor v31.16b, v15.16b, v26.16b + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + shl v23.2d, v31.2d, #0x29 + eor x7, x7, x22, ror #25 + sri v23.2d, v31.2d, #0x17 + bic x22, x22, x24, ror #56 + bic x24, x24, x15, ror #31 + eor v31.16b, v1.16b, v25.16b + eor x22, x22, x15, ror #23 + shl v15.2d, v31.2d, #0x1 + bic x20, x27, x20, ror #48 + sri v15.2d, v31.2d, #0x3f + bic x15, x15, x9, ror #16 + eor x12, x15, x12, ror #58 + eor v31.16b, v8.16b, v28.16b + eor x15, x5, x27, ror #27 + shl v1.2d, v31.2d, #0x37 + eor x5, x20, x11, ror #41 + sri v1.2d, v31.2d, #0x9 + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + eor v31.16b, v16.16b, v25.16b + eor x17, x24, x9, ror #47 + shl v8.2d, v31.2d, #0x2d + mov x24, #0x1 // =1 + sri v8.2d, v31.2d, #0x13 + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + eor v31.16b, v7.16b, v29.16b + bic x24, x29, x1, ror #44 + shl v16.2d, v31.2d, #0x6 + bic x27, x1, x21, ror #50 + sri v16.2d, v31.2d, #0x3a + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + eor v31.16b, v10.16b, v26.16b + ldr x11, [x11] + shl v7.2d, v31.2d, #0x3 + bic x4, x21, x30, ror #57 + sri v7.2d, v31.2d, #0x3d + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + eor v31.16b, v3.16b, v28.16b + bic x9, x14, x6, ror #5 + shl v10.2d, v31.2d, #0x1c + eor x9, x9, x0, ror #43 + sri v10.2d, v31.2d, #0x24 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + eor v31.16b, v18.16b, v28.16b + eor x11, x4, x26, ror #35 + shl v3.2d, v31.2d, #0x15 + eor x4, x0, x16, ror #47 + bic x0, x16, x19, ror #35 + sri v3.2d, v31.2d, #0x2b + eor x16, x27, x30, ror #43 + eor v31.16b, v17.16b, v29.16b + bic x27, x30, x26, ror #42 + shl v18.2d, v31.2d, #0xf + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + sri v18.2d, v31.2d, #0x31 + eor x14, x26, x6, ror #46 + eor v31.16b, v11.16b, v25.16b + eor x6, x27, x29, ror #41 + shl v17.2d, v31.2d, #0xa + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + sri v17.2d, v31.2d, #0x36 + eor x26, x8, x9, ror #57 + eor v31.16b, v9.16b, v27.16b + eor x27, x0, x14, ror #10 + shl v11.2d, v31.2d, #0x14 + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + sri v11.2d, v31.2d, #0x2c + eor x30, x23, x22, ror #50 + eor v31.16b, v22.16b, v29.16b + eor x0, x26, x10, ror #31 + shl v9.2d, v31.2d, #0x3d + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + sri v9.2d, v31.2d, #0x3 + eor x30, x30, x24, ror #34 + eor v31.16b, v14.16b, v27.16b + eor x0, x0, x7, ror #27 + shl v22.2d, v31.2d, #0x27 + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + sri v22.2d, v31.2d, #0x19 + ror x30, x27, #0x3e + eor v31.16b, v20.16b, v26.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + shl v14.2d, v31.2d, #0x12 + eor x16, x30, x16 + sri v14.2d, v31.2d, #0x2e + eor x28, x30, x28, ror #63 + eor v31.16b, v4.16b, v27.16b + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + shl v20.2d, v31.2d, #0x1b + eor x28, x1, x2, ror #61 + sri v20.2d, v31.2d, #0x25 + eor x19, x30, x19, ror #37 + eor v31.16b, v24.16b, v27.16b + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + shl v4.2d, v31.2d, #0xe + eor x26, x26, x0, ror #55 + sri v4.2d, v31.2d, #0x32 + eor x28, x28, x3, ror #39 + eor v31.16b, v21.16b, v25.16b + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + shl v24.2d, v31.2d, #0x2 + eor x0, x0, x29, ror #63 + sri v24.2d, v31.2d, #0x3e + eor x27, x28, x27, ror #61 + eor v31.16b, v5.16b, v26.16b + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + shl v21.2d, v31.2d, #0x24 + eor x29, x30, x20, ror #2 + sri v21.2d, v31.2d, #0x1c + eor x20, x26, x3, ror #39 + eor v31.16b, v6.16b, v25.16b + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + shl v27.2d, v31.2d, #0x2c + eor x3, x28, x21, ror #20 + sri v27.2d, v31.2d, #0x14 + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + bic v31.16b, v7.16b, v11.16b + eor x24, x28, x24, ror #28 + eor v5.16b, v31.16b, v10.16b + eor x1, x30, x17, ror #36 + bic v31.16b, v8.16b, v7.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor v6.16b, v31.16b, v11.16b + eor x8, x27, x8, ror #56 + bic v31.16b, v9.16b, v8.16b + eor x17, x27, x7, ror #19 + eor v7.16b, v31.16b, v7.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bic v31.16b, v10.16b, v9.16b + eor x4, x26, x4, ror #54 + eor v8.16b, v31.16b, v8.16b + eor x0, x0, x12, ror #3 + bic v31.16b, v11.16b, v10.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor v9.16b, v31.16b, v9.16b + eor x26, x26, x5, ror #25 + bic v31.16b, v12.16b, v16.16b + eor x2, x7, x16, ror #39 + eor v10.16b, v31.16b, v15.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + bic v31.16b, v13.16b, v12.16b + eor x7, x7, x22, ror #25 + eor v11.16b, v31.16b, v16.16b + eor x12, x30, x20, ror #58 + bic v31.16b, v14.16b, v13.16b + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor v12.16b, v31.16b, v12.16b + eor x22, x20, x15, ror #23 + bic v31.16b, v15.16b, v14.16b + bic x6, x19, x13, ror #42 + eor v13.16b, v31.16b, v13.16b + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + bic v31.16b, v16.16b, v15.16b + eor x5, x21, x5, ror #21 + eor v14.16b, v31.16b, v14.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + bic v31.16b, v17.16b, v21.16b + bic x21, x21, x25, ror #50 + eor v15.16b, v31.16b, v20.16b + bic x20, x27, x4, ror #25 + bic v31.16b, v18.16b, v17.16b + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + eor v16.16b, v31.16b, v21.16b + eor x21, x17, x25, ror #30 + bic v31.16b, v19.16b, v18.16b + bic x19, x25, x19, ror #57 + eor v17.16b, v31.16b, v17.16b + ldr x25, [sp, #0x18] + eor x17, x10, x9, ror #47 + bic v31.16b, v20.16b, v19.16b + ldr x9, [sp, #0x8] + eor v18.16b, v31.16b, v18.16b + eor x15, x20, x28, ror #27 + bic v31.16b, v21.16b, v20.16b + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + eor v19.16b, v31.16b, v19.16b + bic x20, x11, x27, ror #60 + bic v31.16b, v22.16b, v1.16b + eor x20, x20, x4, ror #21 + eor v20.16b, v31.16b, v0.16b + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + bic v31.16b, v23.16b, v22.16b + ldr x28, [x9, x25, lsl #3] + eor v21.16b, v31.16b, v1.16b + ldr x9, [sp, #0xd0] + bic v31.16b, v24.16b, v23.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + eor v22.16b, v31.16b, v22.16b + cmp x25, #0x17 + bic v31.16b, v0.16b, v24.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor v23.16b, v31.16b, v23.16b + eor x1, x5, x28 + bic v31.16b, v1.16b, v0.16b + eor x5, x4, x11, ror #41 + eor v24.16b, v31.16b, v24.16b + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + bic v31.16b, v2.16b, v27.16b + eor x28, x27, x24, ror #57 + eor v0.16b, v31.16b, v30.16b + bic x27, x24, x9, ror #47 + bic v31.16b, v3.16b, v2.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + eor v1.16b, v31.16b, v27.16b + eor x24, x19, x29, ror #44 + bic v31.16b, v4.16b, v3.16b + bic x29, x3, x29, ror #35 + eor v2.16b, v31.16b, v2.16b + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + bic v31.16b, v30.16b, v4.16b + bic x29, x9, x0, ror #19 + eor v3.16b, v31.16b, v3.16b + bic x14, x14, x8, ror #5 + bic v31.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + eor v4.16b, v31.16b, v4.16b + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor v30.16b, v0.16b, v5.16b + eor v30.16b, v30.16b, v10.16b + eor x26, x8, x9, ror #57 + eor v30.16b, v30.16b, v15.16b + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor v30.16b, v30.16b, v20.16b + eor x26, x26, x6, ror #51 + eor v29.16b, v1.16b, v6.16b + eor x30, x23, x22, ror #50 + eor v29.16b, v29.16b, v11.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor v29.16b, v29.16b, v16.16b + eor x27, x27, x12, ror #5 + eor v29.16b, v29.16b, v21.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor v28.16b, v2.16b, v7.16b + eor x26, x30, x21, ror #26 + eor v28.16b, v28.16b, v12.16b + eor x26, x26, x25, ror #15 + eor v28.16b, v28.16b, v17.16b + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + eor v28.16b, v28.16b, v22.16b + ror x26, x26, #0x3a + eor v27.16b, v3.16b, v8.16b + eor x16, x30, x16 + eor v27.16b, v27.16b, v13.16b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + eor v27.16b, v27.16b, v18.16b + eor x29, x29, x17, ror #36 + eor v27.16b, v27.16b, v23.16b + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor v26.16b, v4.16b, v9.16b + eor x29, x29, x20, ror #2 + eor v26.16b, v26.16b, v14.16b + eor x28, x28, x4, ror #54 + eor v26.16b, v26.16b, v19.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + eor v26.16b, v26.16b, v24.16b + eor x28, x28, x5, ror #25 + add v31.2d, v28.2d, v28.2d + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + sri v31.2d, v28.2d, #0x3f + eor x27, x28, x27, ror #61 + eor v25.16b, v31.16b, v30.16b + eor x13, x0, x13, ror #46 + add v31.2d, v26.2d, v26.2d + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + sri v31.2d, v26.2d, #0x3f + eor x20, x26, x3, ror #39 + eor v28.16b, v31.16b, v28.16b + eor x11, x0, x11, ror #50 + add v31.2d, v29.2d, v29.2d + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + sri v31.2d, v29.2d, #0x3f + eor x21, x26, x1 + eor v26.16b, v31.16b, v26.16b + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + add v31.2d, v27.2d, v27.2d + eor x1, x30, x17, ror #36 + sri v31.2d, v27.2d, #0x3f + eor x14, x0, x14, ror #8 + eor v29.16b, v31.16b, v29.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + add v31.2d, v30.2d, v30.2d + eor x17, x27, x7, ror #19 + sri v31.2d, v30.2d, #0x3f + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor v27.16b, v31.16b, v27.16b + eor x4, x26, x4, ror #54 + eor v30.16b, v0.16b, v26.16b + eor x0, x0, x12, ror #3 + eor v31.16b, v2.16b, v29.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + shl v0.2d, v31.2d, #0x3e + eor x26, x26, x5, ror #25 + sri v0.2d, v31.2d, #0x2 + eor x2, x7, x16, ror #39 + eor v31.16b, v12.16b, v29.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + shl v2.2d, v31.2d, #0x2b + eor x7, x7, x22, ror #25 + sri v2.2d, v31.2d, #0x15 + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor v31.16b, v13.16b, v28.16b + eor x30, x27, x6, ror #43 + shl v12.2d, v31.2d, #0x19 + eor x22, x20, x15, ror #23 + sri v12.2d, v31.2d, #0x27 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + eor v31.16b, v19.16b, v27.16b + bic x5, x13, x17, ror #63 + shl v13.2d, v31.2d, #0x8 + eor x5, x21, x5, ror #21 + sri v13.2d, v31.2d, #0x38 + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v31.16b, v23.16b, v28.16b + bic x21, x21, x25, ror #50 + shl v19.2d, v31.2d, #0x38 + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + sri v19.2d, v31.2d, #0x8 + eor x16, x21, x19, ror #43 + eor v31.16b, v15.16b, v26.16b + eor x21, x17, x25, ror #30 + shl v23.2d, v31.2d, #0x29 + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + sri v23.2d, v31.2d, #0x17 + eor x17, x10, x9, ror #47 + eor v31.16b, v1.16b, v25.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + shl v15.2d, v31.2d, #0x1 + bic x20, x4, x28, ror #2 + sri v15.2d, v31.2d, #0x3f + eor x10, x20, x1, ror #50 + eor v31.16b, v8.16b, v28.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + shl v1.2d, v31.2d, #0x37 + bic x4, x28, x1, ror #48 + sri v1.2d, v31.2d, #0x9 + bic x1, x1, x11, ror #57 + eor v31.16b, v16.16b, v25.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + shl v8.2d, v31.2d, #0x2d + add x25, x25, #0x1 + sri v8.2d, v31.2d, #0x13 + str x25, [sp, #0x18] + cmp x25, #0x17 + eor v31.16b, v7.16b, v29.16b + eor x25, x1, x27, ror #53 + shl v16.2d, v31.2d, #0x6 + bic x27, x30, x26, ror #47 + sri v16.2d, v31.2d, #0x3a + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor v31.16b, v10.16b, v26.16b + eor x11, x19, x13, ror #35 + shl v7.2d, v31.2d, #0x3 + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + sri v7.2d, v31.2d, #0x3d + bic x27, x24, x9, ror #47 + eor v31.16b, v3.16b, v28.16b + bic x19, x23, x3, ror #9 + shl v10.2d, v31.2d, #0x1c + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + sri v10.2d, v31.2d, #0x24 + bic x29, x3, x29, ror #35 + eor v31.16b, v18.16b, v28.16b + eor x13, x13, x9, ror #57 + shl v3.2d, v31.2d, #0x15 + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + sri v3.2d, v31.2d, #0x2b + bic x14, x14, x8, ror #5 + eor v31.16b, v17.16b, v29.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + shl v18.2d, v31.2d, #0xf + bic x23, x8, x23, ror #38 + sri v18.2d, v31.2d, #0x31 + eor x8, x27, x0, ror #2 + eor v31.16b, v11.16b, v25.16b + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + shl v17.2d, v31.2d, #0xa + eor x23, x3, x26, ror #52 + sri v17.2d, v31.2d, #0x36 + eor x3, x29, x30, ror #24 + eor x0, x15, x11, ror #52 + eor v31.16b, v9.16b, v27.16b + eor x0, x0, x13, ror #48 + shl v11.2d, v31.2d, #0x14 + eor x26, x8, x9, ror #57 + sri v11.2d, v31.2d, #0x2c + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + eor v31.16b, v22.16b, v29.16b + eor x26, x26, x6, ror #51 + shl v9.2d, v31.2d, #0x3d + eor x30, x23, x22, ror #50 + sri v9.2d, v31.2d, #0x3 + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + eor v31.16b, v14.16b, v27.16b + eor x27, x27, x12, ror #5 + shl v22.2d, v31.2d, #0x27 + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + sri v22.2d, v31.2d, #0x19 + eor x26, x30, x21, ror #26 + eor v31.16b, v20.16b, v26.16b + eor x26, x26, x25, ror #15 + shl v14.2d, v31.2d, #0x12 + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + sri v14.2d, v31.2d, #0x2e + ror x26, x26, #0x3a + eor v31.16b, v4.16b, v27.16b + eor x16, x30, x16 + shl v20.2d, v31.2d, #0x1b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + sri v20.2d, v31.2d, #0x25 + eor x29, x29, x17, ror #36 + eor v31.16b, v24.16b, v27.16b + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + shl v4.2d, v31.2d, #0xe + eor x29, x29, x20, ror #2 + sri v4.2d, v31.2d, #0x32 + eor x28, x28, x4, ror #54 + eor v31.16b, v21.16b, v25.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + shl v24.2d, v31.2d, #0x2 + eor x28, x28, x5, ror #25 + sri v24.2d, v31.2d, #0x3e + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + eor v31.16b, v5.16b, v26.16b + eor x27, x28, x27, ror #61 + shl v21.2d, v31.2d, #0x24 + eor x13, x0, x13, ror #46 + sri v21.2d, v31.2d, #0x1c + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + eor v31.16b, v6.16b, v25.16b + eor x20, x26, x3, ror #39 + shl v27.2d, v31.2d, #0x2c + eor x11, x0, x11, ror #50 + sri v27.2d, v31.2d, #0x14 + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + bic v31.16b, v7.16b, v11.16b + eor x21, x26, x1 + eor v5.16b, v31.16b, v10.16b + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + bic v31.16b, v8.16b, v7.16b + eor x1, x30, x17, ror #36 + eor v6.16b, v31.16b, v11.16b + eor x14, x0, x14, ror #8 + bic v31.16b, v9.16b, v8.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + eor v7.16b, v31.16b, v7.16b + eor x17, x27, x7, ror #19 + bic v31.16b, v10.16b, v9.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + eor v8.16b, v31.16b, v8.16b + eor x4, x26, x4, ror #54 + bic v31.16b, v11.16b, v10.16b + eor x0, x0, x12, ror #3 + eor v9.16b, v31.16b, v9.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + bic v31.16b, v12.16b, v16.16b + eor x26, x26, x5, ror #25 + eor v10.16b, v31.16b, v15.16b + eor x2, x7, x16, ror #39 + bic v31.16b, v13.16b, v12.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor v11.16b, v31.16b, v16.16b + eor x7, x7, x22, ror #25 + bic v31.16b, v14.16b, v13.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + eor v12.16b, v31.16b, v12.16b + eor x30, x27, x6, ror #43 + bic v31.16b, v15.16b, v14.16b + eor x22, x20, x15, ror #23 + eor v13.16b, v31.16b, v13.16b + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bic v31.16b, v16.16b, v15.16b + bic x5, x13, x17, ror #63 + eor v14.16b, v31.16b, v14.16b + eor x5, x21, x5, ror #21 + bic v31.16b, v17.16b, v21.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v15.16b, v31.16b, v20.16b + bic x21, x21, x25, ror #50 + bic v31.16b, v18.16b, v17.16b + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + eor v16.16b, v31.16b, v21.16b + eor x16, x21, x19, ror #43 + bic v31.16b, v19.16b, v18.16b + eor x21, x17, x25, ror #30 + eor v17.16b, v31.16b, v17.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bic v31.16b, v20.16b, v19.16b + eor x17, x10, x9, ror #47 + eor v18.16b, v31.16b, v18.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + bic v31.16b, v21.16b, v20.16b + bic x20, x4, x28, ror #2 + eor v19.16b, v31.16b, v19.16b + eor x10, x20, x1, ror #50 + bic v31.16b, v22.16b, v1.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + eor v20.16b, v31.16b, v0.16b + bic x4, x28, x1, ror #48 + bic v31.16b, v23.16b, v22.16b + bic x1, x1, x11, ror #57 + eor v21.16b, v31.16b, v1.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bic v31.16b, v24.16b, v23.16b + add x25, x25, #0x1 + eor v22.16b, v31.16b, v22.16b + str x25, [sp, #0x18] + cmp x25, #0x17 + bic v31.16b, v0.16b, v24.16b + eor x25, x1, x27, ror #53 + eor v23.16b, v31.16b, v23.16b + bic x27, x30, x26, ror #47 + bic v31.16b, v1.16b, v0.16b + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + eor v24.16b, v31.16b, v24.16b + eor x11, x19, x13, ror #35 + bic v31.16b, v2.16b, v27.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v0.16b, v31.16b, v30.16b + bic x27, x24, x9, ror #47 + bic v31.16b, v3.16b, v2.16b + bic x19, x23, x3, ror #9 + eor v1.16b, v31.16b, v27.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v4.16b, v3.16b + bic x29, x3, x29, ror #35 + eor v2.16b, v31.16b, v2.16b + eor x13, x13, x9, ror #57 + bic v31.16b, v30.16b, v4.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + eor v3.16b, v31.16b, v3.16b + bic x14, x14, x8, ror #5 + bic v31.16b, v27.16b, v30.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + eor v4.16b, v31.16b, v4.16b + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_loop_end: + b.le Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x30, [sp, #0x20] + cmp x30, #0x1 + b.eq Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_done + mov x30, #0x1 // =1 + str x30, [sp, #0x20] + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x0, x0, #0x258 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x258 + b Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_initial + +Lmlk_keccak_f1600_x4_v8a_scalar_hybrid_done: + ldr x0, [sp] + add x0, x0, #0x258 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x258 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm) + +#endif /* MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S new file mode 100644 index 0000000000..c17267c605 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S @@ -0,0 +1,989 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2021-2022 Arm Limited + * Copyright (c) 2022 Matthias Kannwischer + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// Author: Hanno Becker +// Author: Matthias Kannwischer + +/*yaml + Name: keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm + Description: AArch64 hybrid scalar/vector implementation of Keccak-f[1600] permutation for four sequential states with ARMv8.4-A optimizations + Signature: void mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm(uint64_t state[100], const uint64_t rc[24]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON, SHA3] + x0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t state[100] + description: Four sequential Keccak states (state0[25], state1[25], state2[25], state3[25]) + x1: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + Stack: + bytes: 224 + description: register preservation and temporary storage +*/ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#if defined(__ARM_FEATURE_SHA3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/aarch64/src/keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0xe0 + .cfi_adjust_cfa_offset 0xe0 + stp x19, x20, [sp, #0x30] + .cfi_rel_offset x19, 0x30 + .cfi_rel_offset x20, 0x38 + stp x21, x22, [sp, #0x40] + .cfi_rel_offset x21, 0x40 + .cfi_rel_offset x22, 0x48 + stp x23, x24, [sp, #0x50] + .cfi_rel_offset x23, 0x50 + .cfi_rel_offset x24, 0x58 + stp x25, x26, [sp, #0x60] + .cfi_rel_offset x25, 0x60 + .cfi_rel_offset x26, 0x68 + stp x27, x28, [sp, #0x70] + .cfi_rel_offset x27, 0x70 + .cfi_rel_offset x28, 0x78 + stp x29, x30, [sp, #0x80] + .cfi_rel_offset x29, 0x80 + .cfi_rel_offset x30, 0x88 + stp d8, d9, [sp, #0x90] + .cfi_rel_offset d8, 0x90 + .cfi_rel_offset d9, 0x98 + stp d10, d11, [sp, #0xa0] + .cfi_rel_offset d10, 0xa0 + .cfi_rel_offset d11, 0xa8 + stp d12, d13, [sp, #0xb0] + .cfi_rel_offset d12, 0xb0 + .cfi_rel_offset d13, 0xb8 + stp d14, d15, [sp, #0xc0] + .cfi_rel_offset d14, 0xc0 + .cfi_rel_offset d15, 0xc8 + mov x29, x1 + mov x30, #0x0 // =0 + str x30, [sp, #0x20] + str x29, [sp, #0x8] + str x29, [sp, #0x10] + str x0, [sp] + add x4, x0, #0xc8 + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v0.2d, v25.2d, v27.2d + trn2 v1.2d, v25.2d, v27.2d + trn1 v2.2d, v26.2d, v28.2d + trn2 v3.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v4.2d, v25.2d, v27.2d + trn2 v5.2d, v25.2d, v27.2d + trn1 v6.2d, v26.2d, v28.2d + trn2 v7.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v8.2d, v25.2d, v27.2d + trn2 v9.2d, v25.2d, v27.2d + trn1 v10.2d, v26.2d, v28.2d + trn2 v11.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v12.2d, v25.2d, v27.2d + trn2 v13.2d, v25.2d, v27.2d + trn1 v14.2d, v26.2d, v28.2d + trn2 v15.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v16.2d, v25.2d, v27.2d + trn2 v17.2d, v25.2d, v27.2d + trn1 v18.2d, v26.2d, v28.2d + trn2 v19.2d, v26.2d, v28.2d + ldp q25, q26, [x0], #0x20 + ld1 { v27.2d, v28.2d }, [x4], #32 + trn1 v20.2d, v25.2d, v27.2d + trn2 v21.2d, v25.2d, v27.2d + trn1 v22.2d, v26.2d, v28.2d + trn2 v23.2d, v26.2d, v28.2d + ldr d25, [x0] + ldr d27, [x4] + trn1 v24.2d, v25.2d, v27.2d + sub x0, x0, #0xc0 + add x0, x0, #0x190 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x190 + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial: + eor x30, x24, x25 + eor x27, x9, x10 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor v30.16b, v30.16b, v15.16b + eor x0, x30, x21 + eor x26, x27, x6 + eor v30.16b, v30.16b, v20.16b + eor x27, x26, x7 + eor x29, x0, x22 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x26, x29, x23 + eor x29, x4, x5 + eor v29.16b, v29.16b, v16.16b + eor x30, x29, x1 + eor x0, x27, x8 + eor v29.16b, v29.16b, v21.16b + eor x29, x30, x2 + eor x30, x19, x20 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x30, x30, x16 + eor x27, x26, x0, ror #63 + eor v28.16b, v28.16b, v17.16b + eor x4, x4, x27 + eor x30, x30, x17 + eor v28.16b, v28.16b, v22.16b + eor x30, x30, x28 + eor x29, x29, x3 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x0, x0, x30, ror #63 + eor x30, x30, x29, ror #63 + eor v27.16b, v27.16b, v18.16b + eor x22, x22, x30 + eor v27.16b, v27.16b, v23.16b + eor x23, x23, x30 + str x23, [sp, #0xd0] + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x23, x14, x15 + eor x14, x14, x0 + eor v26.16b, v26.16b, v19.16b + eor x23, x23, x11 + eor x15, x15, x0 + eor v26.16b, v26.16b, v24.16b + eor x1, x1, x27 + eor x23, x23, x12 + rax1 v25.2d, v30.2d, v28.2d + eor x23, x23, x13 + eor x11, x11, x0 + add v31.2d, v26.2d, v26.2d + eor x29, x29, x23, ror #63 + eor x23, x23, x26, ror #63 + sri v31.2d, v26.2d, #0x3f + eor x26, x13, x0 + eor x13, x28, x23 + eor v28.16b, v31.16b, v28.16b + eor x28, x24, x30 + eor x24, x16, x23 + rax1 v26.2d, v26.2d, v29.2d + eor x16, x21, x30 + eor x21, x25, x30 + add v31.2d, v27.2d, v27.2d + eor x30, x19, x23 + sri v31.2d, v27.2d, #0x3f + eor x19, x20, x23 + eor x20, x17, x23 + eor v29.16b, v31.16b, v29.16b + eor x17, x12, x0 + eor x0, x2, x27 + rax1 v27.2d, v27.2d, v30.2d + eor x2, x6, x29 + eor x6, x8, x29 + eor v30.16b, v0.16b, v26.16b + bic x8, x28, x13, ror #47 + eor x12, x3, x27 + eor v31.16b, v2.16b, v29.16b + bic x3, x13, x17, ror #19 + eor x5, x5, x27 + shl v0.2d, v31.2d, #0x3e + ldr x27, [sp, #0xd0] + bic x25, x17, x2, ror #5 + sri v0.2d, v31.2d, #0x2 + eor x9, x9, x29 + eor x23, x25, x5, ror #52 + xar v2.2d, v12.2d, v29.2d, #0x15 + eor x3, x3, x2, ror #24 + eor x8, x8, x17, ror #2 + eor v31.16b, v13.16b, v28.16b + eor x17, x10, x29 + bic x25, x12, x22, ror #47 + shl v12.2d, v31.2d, #0x19 + eor x29, x7, x29 + bic x10, x4, x27, ror #2 + sri v12.2d, v31.2d, #0x27 + bic x7, x5, x28, ror #10 + xar v13.2d, v19.2d, v27.2d, #0x38 + eor x10, x10, x20, ror #50 + eor x13, x7, x13, ror #57 + eor v31.16b, v23.16b, v28.16b + bic x7, x2, x5, ror #47 + eor x2, x25, x24, ror #39 + shl v19.2d, v31.2d, #0x38 + bic x25, x20, x11, ror #57 + bic x5, x17, x4, ror #25 + sri v19.2d, v31.2d, #0x8 + eor x25, x25, x17, ror #53 + bic x17, x11, x17, ror #60 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x28, x7, x28, ror #57 + bic x7, x9, x12, ror #42 + eor v31.16b, v1.16b, v25.16b + eor x7, x7, x22, ror #25 + bic x22, x22, x24, ror #56 + shl v15.2d, v31.2d, #0x1 + bic x24, x24, x15, ror #31 + eor x22, x22, x15, ror #23 + sri v15.2d, v31.2d, #0x3f + bic x20, x27, x20, ror #48 + bic x15, x15, x9, ror #16 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x12, x15, x12, ror #58 + eor x15, x5, x27, ror #27 + eor v31.16b, v16.16b, v25.16b + eor x5, x20, x11, ror #41 + shl v8.2d, v31.2d, #0x2d + ldr x11, [sp, #0x8] + eor x20, x17, x4, ror #21 + sri v8.2d, v31.2d, #0x13 + eor x17, x24, x9, ror #47 + mov x24, #0x1 // =1 + xar v16.2d, v7.2d, v29.2d, #0x3a + bic x9, x0, x16, ror #9 + str x24, [sp, #0x18] + eor v31.16b, v10.16b, v26.16b + bic x24, x29, x1, ror #44 + bic x27, x1, x21, ror #50 + shl v7.2d, v31.2d, #0x3 + bic x4, x26, x29, ror #63 + eor x1, x1, x4, ror #21 + sri v7.2d, v31.2d, #0x3d + ldr x11, [x11] + bic x4, x21, x30, ror #57 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x21, x24, x21, ror #30 + eor x24, x9, x19, ror #44 + eor v31.16b, v18.16b, v28.16b + bic x9, x14, x6, ror #5 + eor x9, x9, x0, ror #43 + shl v3.2d, v31.2d, #0x15 + bic x0, x6, x0, ror #38 + eor x1, x1, x11 + sri v3.2d, v31.2d, #0x2b + eor x11, x4, x26, ror #35 + eor x4, x0, x16, ror #47 + xar v18.2d, v17.2d, v29.2d, #0x31 + bic x0, x16, x19, ror #35 + eor v31.16b, v11.16b, v25.16b + eor x16, x27, x30, ror #43 + bic x27, x30, x26, ror #42 + shl v17.2d, v31.2d, #0xa + bic x26, x19, x14, ror #41 + eor x19, x0, x14, ror #12 + sri v17.2d, v31.2d, #0x36 + eor x14, x26, x6, ror #46 + eor x6, x27, x29, ror #41 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor v31.16b, v22.16b, v29.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + shl v9.2d, v31.2d, #0x3d + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + sri v9.2d, v31.2d, #0x3 + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor v31.16b, v20.16b, v26.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + shl v14.2d, v31.2d, #0x12 + eor x26, x30, x21, ror #26 + sri v14.2d, v31.2d, #0x2e + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + eor v31.16b, v24.16b, v27.16b + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + shl v4.2d, v31.2d, #0xe + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + sri v4.2d, v31.2d, #0x32 + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + eor v31.16b, v5.16b, v26.16b + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + shl v21.2d, v31.2d, #0x24 + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + sri v21.2d, v31.2d, #0x1c + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + bic v31.16b, v7.16b, v11.16b + eor x29, x30, x20, ror #2 + eor v5.16b, v31.16b, v10.16b + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + bic v31.16b, v9.16b, v8.16b + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + eor v7.16b, v31.16b, v7.16b + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + bic v31.16b, v11.16b, v10.16b + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + eor v9.16b, v31.16b, v9.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + bic v31.16b, v13.16b, v12.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + eor v11.16b, v31.16b, v16.16b + eor x26, x26, x5, ror #25 + bcax v12.16b, v12.16b, v14.16b, v13.16b + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + bic v31.16b, v15.16b, v14.16b + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor v13.16b, v31.16b, v13.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + bic v31.16b, v16.16b, v15.16b + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + eor v14.16b, v31.16b, v14.16b + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + bcax v15.16b, v20.16b, v17.16b, v21.16b + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + bic v31.16b, v18.16b, v17.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + eor v16.16b, v31.16b, v21.16b + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + bcax v17.16b, v17.16b, v19.16b, v18.16b + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + bic v31.16b, v20.16b, v19.16b + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + eor v18.16b, v31.16b, v18.16b + ldr x25, [sp, #0x18] + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x17, x10, x9, ror #47 + ldr x9, [sp, #0x8] + bic v31.16b, v22.16b, v1.16b + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + eor v20.16b, v31.16b, v0.16b + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + bcax v21.16b, v1.16b, v23.16b, v22.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + bic v31.16b, v24.16b, v23.16b + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + eor v22.16b, v31.16b, v22.16b + ldr x9, [sp, #0xd0] + add x25, x25, #0x1 + bcax v23.16b, v23.16b, v0.16b, v24.16b + str x25, [sp, #0x18] + cmp x25, #0x17 + bic v31.16b, v1.16b, v0.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + eor v24.16b, v31.16b, v24.16b + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x11, x19, x13, ror #35 + bic v31.16b, v3.16b, v2.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v1.16b, v31.16b, v27.16b + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bcax v2.16b, v2.16b, v4.16b, v3.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v30.16b, v4.16b + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor v3.16b, v31.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop: + eor x0, x15, x11, ror #52 + eor x0, x0, x13, ror #48 + eor3 v30.16b, v0.16b, v5.16b, v10.16b + eor v30.16b, v30.16b, v15.16b + eor x26, x8, x9, ror #57 + eor x27, x0, x14, ror #10 + eor v30.16b, v30.16b, v20.16b + eor x29, x16, x28, ror #63 + eor x26, x26, x6, ror #51 + eor3 v29.16b, v1.16b, v6.16b, v11.16b + eor x30, x23, x22, ror #50 + eor x0, x26, x10, ror #31 + eor v29.16b, v29.16b, v16.16b + eor x29, x29, x19, ror #37 + eor x27, x27, x12, ror #5 + eor v29.16b, v29.16b, v21.16b + eor x30, x30, x24, ror #34 + eor x0, x0, x7, ror #27 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x26, x30, x21, ror #26 + eor x26, x26, x25, ror #15 + eor v28.16b, v28.16b, v17.16b + ror x30, x27, #0x3e + eor x30, x30, x26, ror #57 + eor v28.16b, v28.16b, v22.16b + ror x26, x26, #0x3a + eor x16, x30, x16 + eor3 v27.16b, v3.16b, v8.16b, v13.16b + eor x28, x30, x28, ror #63 + str x28, [sp, #0xd0] + eor v27.16b, v27.16b, v18.16b + eor x29, x29, x17, ror #36 + eor x28, x1, x2, ror #61 + eor v27.16b, v27.16b, v23.16b + eor x19, x30, x19, ror #37 + eor x29, x29, x20, ror #2 + eor3 v26.16b, v4.16b, v9.16b, v14.16b + eor x28, x28, x4, ror #54 + eor x26, x26, x0, ror #55 + eor v26.16b, v26.16b, v19.16b + eor x28, x28, x3, ror #39 + eor x28, x28, x5, ror #25 + eor v26.16b, v26.16b, v24.16b + ror x0, x0, #0x38 + eor x0, x0, x29, ror #63 + rax1 v25.2d, v30.2d, v28.2d + eor x27, x28, x27, ror #61 + eor x13, x0, x13, ror #46 + add v31.2d, v26.2d, v26.2d + eor x28, x29, x28, ror #63 + eor x29, x30, x20, ror #2 + sri v31.2d, v26.2d, #0x3f + eor x20, x26, x3, ror #39 + eor x11, x0, x11, ror #50 + eor v28.16b, v31.16b, v28.16b + eor x25, x28, x25, ror #9 + eor x3, x28, x21, ror #20 + rax1 v26.2d, v26.2d, v29.2d + eor x21, x26, x1 + add v31.2d, v27.2d, v27.2d + eor x9, x27, x9, ror #49 + eor x24, x28, x24, ror #28 + sri v31.2d, v27.2d, #0x3f + eor x1, x30, x17, ror #36 + eor x14, x0, x14, ror #8 + eor v29.16b, v31.16b, v29.16b + eor x22, x28, x22, ror #44 + eor x8, x27, x8, ror #56 + rax1 v27.2d, v27.2d, v30.2d + eor x17, x27, x7, ror #19 + eor x15, x0, x15, ror #62 + eor v30.16b, v0.16b, v26.16b + bic x7, x20, x22, ror #47 + eor x4, x26, x4, ror #54 + eor v31.16b, v2.16b, v29.16b + eor x0, x0, x12, ror #3 + eor x28, x28, x23, ror #58 + shl v0.2d, v31.2d, #0x3e + eor x23, x26, x2, ror #61 + eor x26, x26, x5, ror #25 + sri v0.2d, v31.2d, #0x2 + eor x2, x7, x16, ror #39 + bic x7, x9, x20, ror #42 + xar v2.2d, v12.2d, v29.2d, #0x15 + bic x30, x15, x9, ror #16 + eor x7, x7, x22, ror #25 + eor v31.16b, v13.16b, v28.16b + eor x12, x30, x20, ror #58 + bic x20, x22, x16, ror #56 + shl v12.2d, v31.2d, #0x19 + eor x30, x27, x6, ror #43 + eor x22, x20, x15, ror #23 + sri v12.2d, v31.2d, #0x27 + bic x6, x19, x13, ror #42 + eor x6, x6, x17, ror #41 + xar v13.2d, v19.2d, v27.2d, #0x38 + bic x5, x13, x17, ror #63 + eor x5, x21, x5, ror #21 + eor v31.16b, v23.16b, v28.16b + bic x17, x17, x21, ror #44 + eor x27, x27, x10, ror #23 + shl v19.2d, v31.2d, #0x38 + bic x21, x21, x25, ror #50 + bic x20, x27, x4, ror #25 + sri v19.2d, v31.2d, #0x8 + bic x10, x16, x15, ror #31 + eor x16, x21, x19, ror #43 + xar v23.2d, v15.2d, v26.2d, #0x17 + eor x21, x17, x25, ror #30 + bic x19, x25, x19, ror #57 + eor v31.16b, v1.16b, v25.16b + ldr x25, [sp, #0x18] + eor x17, x10, x9, ror #47 + shl v15.2d, v31.2d, #0x1 + ldr x9, [sp, #0x8] + sri v15.2d, v31.2d, #0x3f + eor x15, x20, x28, ror #27 + bic x20, x4, x28, ror #2 + xar v1.2d, v8.2d, v28.2d, #0x9 + eor x10, x20, x1, ror #50 + bic x20, x11, x27, ror #60 + eor v31.16b, v16.16b, v25.16b + eor x20, x20, x4, ror #21 + bic x4, x28, x1, ror #48 + shl v8.2d, v31.2d, #0x2d + bic x1, x1, x11, ror #57 + ldr x28, [x9, x25, lsl #3] + sri v8.2d, v31.2d, #0x13 + ldr x9, [sp, #0xd0] + add x25, x25, #0x1 + xar v16.2d, v7.2d, v29.2d, #0x3a + str x25, [sp, #0x18] + cmp x25, #0x17 + eor v31.16b, v10.16b, v26.16b + eor x25, x1, x27, ror #53 + bic x27, x30, x26, ror #47 + shl v7.2d, v31.2d, #0x3 + eor x1, x5, x28 + eor x5, x4, x11, ror #41 + sri v7.2d, v31.2d, #0x3d + eor x11, x19, x13, ror #35 + bic x13, x26, x24, ror #10 + xar v10.2d, v3.2d, v28.2d, #0x24 + eor x28, x27, x24, ror #57 + bic x27, x24, x9, ror #47 + eor v31.16b, v18.16b, v28.16b + bic x19, x23, x3, ror #9 + bic x4, x29, x14, ror #41 + shl v3.2d, v31.2d, #0x15 + eor x24, x19, x29, ror #44 + bic x29, x3, x29, ror #35 + sri v3.2d, v31.2d, #0x2b + eor x13, x13, x9, ror #57 + eor x19, x29, x14, ror #12 + xar v18.2d, v17.2d, v29.2d, #0x31 + bic x29, x9, x0, ror #19 + bic x14, x14, x8, ror #5 + eor v31.16b, v11.16b, v25.16b + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + shl v17.2d, v31.2d, #0xa + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + sri v17.2d, v31.2d, #0x36 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + xar v11.2d, v9.2d, v27.2d, #0x2c + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + eor v31.16b, v22.16b, v29.16b + eor x0, x15, x11, ror #52 + shl v9.2d, v31.2d, #0x3d + eor x0, x0, x13, ror #48 + eor x26, x8, x9, ror #57 + sri v9.2d, v31.2d, #0x3 + eor x27, x0, x14, ror #10 + eor x29, x16, x28, ror #63 + xar v22.2d, v14.2d, v27.2d, #0x19 + eor x26, x26, x6, ror #51 + eor x30, x23, x22, ror #50 + eor v31.16b, v20.16b, v26.16b + eor x0, x26, x10, ror #31 + eor x29, x29, x19, ror #37 + shl v14.2d, v31.2d, #0x12 + eor x27, x27, x12, ror #5 + eor x30, x30, x24, ror #34 + sri v14.2d, v31.2d, #0x2e + eor x0, x0, x7, ror #27 + eor x26, x30, x21, ror #26 + xar v20.2d, v4.2d, v27.2d, #0x25 + eor x26, x26, x25, ror #15 + ror x30, x27, #0x3e + eor v31.16b, v24.16b, v27.16b + eor x30, x30, x26, ror #57 + ror x26, x26, #0x3a + shl v4.2d, v31.2d, #0xe + eor x16, x30, x16 + eor x28, x30, x28, ror #63 + sri v4.2d, v31.2d, #0x32 + str x28, [sp, #0xd0] + eor x29, x29, x17, ror #36 + xar v24.2d, v21.2d, v25.2d, #0x3e + eor x28, x1, x2, ror #61 + eor x19, x30, x19, ror #37 + eor v31.16b, v5.16b, v26.16b + eor x29, x29, x20, ror #2 + eor x28, x28, x4, ror #54 + shl v21.2d, v31.2d, #0x24 + eor x26, x26, x0, ror #55 + eor x28, x28, x3, ror #39 + sri v21.2d, v31.2d, #0x1c + eor x28, x28, x5, ror #25 + ror x0, x0, #0x38 + xar v27.2d, v6.2d, v25.2d, #0x14 + eor x0, x0, x29, ror #63 + eor x27, x28, x27, ror #61 + bic v31.16b, v7.16b, v11.16b + eor x13, x0, x13, ror #46 + eor x28, x29, x28, ror #63 + eor v5.16b, v31.16b, v10.16b + eor x29, x30, x20, ror #2 + eor x20, x26, x3, ror #39 + bcax v6.16b, v11.16b, v8.16b, v7.16b + eor x11, x0, x11, ror #50 + eor x25, x28, x25, ror #9 + bic v31.16b, v9.16b, v8.16b + eor x3, x28, x21, ror #20 + eor v7.16b, v31.16b, v7.16b + eor x21, x26, x1 + eor x9, x27, x9, ror #49 + bcax v8.16b, v8.16b, v10.16b, v9.16b + eor x24, x28, x24, ror #28 + eor x1, x30, x17, ror #36 + bic v31.16b, v11.16b, v10.16b + eor x14, x0, x14, ror #8 + eor x22, x28, x22, ror #44 + eor v9.16b, v31.16b, v9.16b + eor x8, x27, x8, ror #56 + eor x17, x27, x7, ror #19 + bcax v10.16b, v15.16b, v12.16b, v16.16b + eor x15, x0, x15, ror #62 + bic x7, x20, x22, ror #47 + bic v31.16b, v13.16b, v12.16b + eor x4, x26, x4, ror #54 + eor x0, x0, x12, ror #3 + eor v11.16b, v31.16b, v16.16b + eor x28, x28, x23, ror #58 + eor x23, x26, x2, ror #61 + bcax v12.16b, v12.16b, v14.16b, v13.16b + eor x26, x26, x5, ror #25 + eor x2, x7, x16, ror #39 + bic v31.16b, v15.16b, v14.16b + bic x7, x9, x20, ror #42 + bic x30, x15, x9, ror #16 + eor v13.16b, v31.16b, v13.16b + eor x7, x7, x22, ror #25 + eor x12, x30, x20, ror #58 + bic v31.16b, v16.16b, v15.16b + bic x20, x22, x16, ror #56 + eor x30, x27, x6, ror #43 + eor v14.16b, v31.16b, v14.16b + eor x22, x20, x15, ror #23 + bic x6, x19, x13, ror #42 + bcax v15.16b, v20.16b, v17.16b, v21.16b + eor x6, x6, x17, ror #41 + bic x5, x13, x17, ror #63 + bic v31.16b, v18.16b, v17.16b + eor x5, x21, x5, ror #21 + bic x17, x17, x21, ror #44 + eor v16.16b, v31.16b, v21.16b + eor x27, x27, x10, ror #23 + bic x21, x21, x25, ror #50 + bcax v17.16b, v17.16b, v19.16b, v18.16b + bic x20, x27, x4, ror #25 + bic x10, x16, x15, ror #31 + bic v31.16b, v20.16b, v19.16b + eor x16, x21, x19, ror #43 + eor x21, x17, x25, ror #30 + eor v18.16b, v31.16b, v18.16b + bic x19, x25, x19, ror #57 + ldr x25, [sp, #0x18] + bcax v19.16b, v19.16b, v21.16b, v20.16b + eor x17, x10, x9, ror #47 + bic v31.16b, v22.16b, v1.16b + ldr x9, [sp, #0x8] + eor x15, x20, x28, ror #27 + eor v20.16b, v31.16b, v0.16b + bic x20, x4, x28, ror #2 + eor x10, x20, x1, ror #50 + bcax v21.16b, v1.16b, v23.16b, v22.16b + bic x20, x11, x27, ror #60 + eor x20, x20, x4, ror #21 + bic v31.16b, v24.16b, v23.16b + bic x4, x28, x1, ror #48 + bic x1, x1, x11, ror #57 + eor v22.16b, v31.16b, v22.16b + ldr x28, [x9, x25, lsl #3] + ldr x9, [sp, #0xd0] + bcax v23.16b, v23.16b, v0.16b, v24.16b + add x25, x25, #0x1 + str x25, [sp, #0x18] + bic v31.16b, v1.16b, v0.16b + cmp x25, #0x17 + eor x25, x1, x27, ror #53 + eor v24.16b, v31.16b, v24.16b + bic x27, x30, x26, ror #47 + eor x1, x5, x28 + bcax v0.16b, v30.16b, v2.16b, v27.16b + eor x5, x4, x11, ror #41 + eor x11, x19, x13, ror #35 + bic v31.16b, v3.16b, v2.16b + bic x13, x26, x24, ror #10 + eor x28, x27, x24, ror #57 + eor v1.16b, v31.16b, v27.16b + bic x27, x24, x9, ror #47 + bic x19, x23, x3, ror #9 + bcax v2.16b, v2.16b, v4.16b, v3.16b + bic x4, x29, x14, ror #41 + eor x24, x19, x29, ror #44 + bic v31.16b, v30.16b, v4.16b + bic x29, x3, x29, ror #35 + eor x13, x13, x9, ror #57 + eor v3.16b, v31.16b, v3.16b + eor x19, x29, x14, ror #12 + bic x29, x9, x0, ror #19 + bcax v4.16b, v4.16b, v27.16b, v30.16b + bic x14, x14, x8, ror #5 + eor x9, x14, x23, ror #43 + eor x14, x4, x8, ror #46 + bic x23, x8, x23, ror #38 + eor x8, x27, x0, ror #2 + eor x4, x23, x3, ror #47 + bic x3, x0, x30, ror #5 + eor x23, x3, x26, ror #52 + eor x3, x29, x30, ror #24 + ldr x30, [sp, #0x10] + ld1r { v28.2d }, [x30], #8 + str x30, [sp, #0x10] + eor v0.16b, v0.16b, v28.16b + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop_end: + b.le Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_loop + ror x2, x2, #0x3d + ror x3, x3, #0x27 + ror x4, x4, #0x36 + ror x5, x5, #0x19 + ror x6, x6, #0x2b + ror x7, x7, #0x13 + ror x8, x8, #0x38 + ror x9, x9, #0x31 + ror x10, x10, #0x17 + ror x11, x11, #0x32 + ror x12, x12, #0x3 + ror x13, x13, #0x2e + ror x14, x14, #0x8 + ror x15, x15, #0x3e + ror x17, x17, #0x24 + ror x28, x28, #0x3f + ror x19, x19, #0x25 + ror x20, x20, #0x2 + ror x21, x21, #0x14 + ror x22, x22, #0x2c + ror x23, x23, #0x3a + ror x24, x24, #0x1c + ror x25, x25, #0x9 + ldr x30, [sp, #0x20] + cmp x30, #0x1 + b.eq Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done + mov x30, #0x1 // =1 + str x30, [sp, #0x20] + ldr x0, [sp] + add x0, x0, #0x190 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x190 + add x0, x0, #0x258 + ldp x1, x6, [x0] + ldp x11, x16, [x0, #0x10] + ldp x21, x2, [x0, #0x20] + ldp x7, x12, [x0, #0x30] + ldp x17, x22, [x0, #0x40] + ldp x3, x8, [x0, #0x50] + ldp x13, x28, [x0, #0x60] + ldp x23, x4, [x0, #0x70] + ldp x9, x14, [x0, #0x80] + ldp x19, x24, [x0, #0x90] + ldp x5, x10, [x0, #0xa0] + ldp x15, x20, [x0, #0xb0] + ldr x25, [x0, #0xc0] + sub x0, x0, #0x258 + b Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_initial + +Lmlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_done: + ldr x0, [sp] + add x0, x0, #0x258 + stp x1, x6, [x0] + stp x11, x16, [x0, #0x10] + stp x21, x2, [x0, #0x20] + stp x7, x12, [x0, #0x30] + stp x17, x22, [x0, #0x40] + stp x3, x8, [x0, #0x50] + stp x13, x28, [x0, #0x60] + stp x23, x4, [x0, #0x70] + stp x9, x14, [x0, #0x80] + stp x19, x24, [x0, #0x90] + stp x5, x10, [x0, #0xa0] + stp x15, x20, [x0, #0xb0] + str x25, [x0, #0xc0] + sub x0, x0, #0x258 + add x4, x0, #0xc8 + trn1 v25.2d, v0.2d, v1.2d + trn1 v26.2d, v2.2d, v3.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v0.2d, v1.2d + trn2 v28.2d, v2.2d, v3.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v4.2d, v5.2d + trn1 v26.2d, v6.2d, v7.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v4.2d, v5.2d + trn2 v28.2d, v6.2d, v7.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v8.2d, v9.2d + trn1 v26.2d, v10.2d, v11.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v8.2d, v9.2d + trn2 v28.2d, v10.2d, v11.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v12.2d, v13.2d + trn1 v26.2d, v14.2d, v15.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v12.2d, v13.2d + trn2 v28.2d, v14.2d, v15.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v16.2d, v17.2d + trn1 v26.2d, v18.2d, v19.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v16.2d, v17.2d + trn2 v28.2d, v18.2d, v19.2d + st1 { v27.2d, v28.2d }, [x4], #32 + trn1 v25.2d, v20.2d, v21.2d + trn1 v26.2d, v22.2d, v23.2d + stp q25, q26, [x0], #0x20 + trn2 v27.2d, v20.2d, v21.2d + trn2 v28.2d, v22.2d, v23.2d + st1 { v27.2d, v28.2d }, [x4], #32 + str d24, [x0] + trn2 v25.2d, v24.2d, v24.2d + str d25, [x4] + ldp d8, d9, [sp, #0x90] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0xa0] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0xb0] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0xc0] + .cfi_restore d14 + .cfi_restore d15 + ldp x19, x20, [sp, #0x30] + .cfi_restore x19 + .cfi_restore x20 + ldp x21, x22, [sp, #0x40] + .cfi_restore x21 + .cfi_restore x22 + ldp x23, x24, [sp, #0x50] + .cfi_restore x23 + .cfi_restore x24 + ldp x25, x26, [sp, #0x60] + .cfi_restore x25 + .cfi_restore x26 + ldp x27, x28, [sp, #0x70] + .cfi_restore x27 + .cfi_restore x28 + ldp x29, x30, [sp, #0x80] + .cfi_restore x29 + .cfi_restore x30 + add sp, sp, #0xe0 + .cfi_adjust_cfa_offset -0xe0 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm) + +#endif /* __ARM_FEATURE_SHA3 */ + +#endif /* MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c new file mode 100644 index 0000000000..12a897f04b --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/src/keccakf1600_round_constants.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" + +#if (defined(MLK_FIPS202_AARCH64_NEED_X1_SCALAR) || \ + defined(MLK_FIPS202_AARCH64_NEED_X1_V84A) || \ + defined(MLK_FIPS202_AARCH64_NEED_X2_V84A) || \ + defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID) || \ + defined(MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID)) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_aarch64.h" + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t + mlk_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +#else /* (MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \ + MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \ + || MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202_aarch64_round_constants) + +#endif /* !((MLK_FIPS202_AARCH64_NEED_X1_SCALAR || \ + MLK_FIPS202_AARCH64_NEED_X1_V84A || MLK_FIPS202_AARCH64_NEED_X2_V84A \ + || MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID || \ + MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID) && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h new file mode 100644 index 0000000000..ccf805b002 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_scalar.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X1 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X1_SCALAR + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +{ + mlk_keccak_f1600_x1_scalar_aarch64_asm(state, + mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_SCALAR_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h new file mode 100644 index 0000000000..72d538a66b --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x1_v84a.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H +#define MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X1 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X1_V84A + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || + !mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x1_v84a_aarch64_asm(state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X1_V84A_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h new file mode 100644 index 0000000000..5848454828 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x2_v84a.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H +#define MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X2_V84A + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || + !mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 0 * 25, + mlk_keccakf1600_round_constants); + mlk_keccak_f1600_x2_v84a_aarch64_asm(state + 2 * 25, + mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X2_V84A_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h new file mode 100644 index 0000000000..e3f84ad52d --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X4_V8A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_v8a_scalar_hybrid_aarch64_asm( + state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_SCALAR_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h new file mode 100644 index 0000000000..bd112cc5d9 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/aarch64/x4_v8a_v84a_scalar.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H +#define MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H + +#if !defined(__ARM_FEATURE_SHA3) +#error This backend can only be used if SHA3 extensions are available. +#endif + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +/* Guard for assembly file */ +#define MLK_FIPS202_AARCH64_NEED_X4_V8A_V84A_SCALAR_HYBRID + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_aarch64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || + !mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_SHA3)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_v8a_v84a_scalar_hybrid_aarch64_asm( + state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_AARCH64_X4_V8A_V84A_SCALAR_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/api.h b/src/mlkem-native/mlkem/src/fips202/native/api.h new file mode 100644 index 0000000000..8ef7383e25 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/api.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_API_H +#define MLK_FIPS202_NATIVE_API_H +/* + * FIPS-202 native interface + * + * This header is primarily for documentation purposes. + * It should not be included by backend implementations. + */ + +#include "../../cbmc.h" + +/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */ +#define MLK_NATIVE_FUNC_SUCCESS (0) +/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that + * the target/parameters are unsupported; typically, this would be because of + * dependencies on CPU features not detected on the host CPU. In this case, + * the frontend falls back to the default C implementation. */ +#define MLK_NATIVE_FUNC_FALLBACK (-1) + +/* + * This is the C<->native interface allowing for the drop-in + * of custom Keccak-F1600 implementations. + * + * A _backend_ is a specific implementation of parts of this interface. + * + * You can replace 1-fold or 4-fold batched Keccak-F1600. + * To enable, set MLK_USE_NATIVE_FIPS202_X1 or MLK_USE_NATIVE_FIPS202_X4 + * in your backend, and define the inline wrappers mlk_keccak_f1600_x1_native() + * and/or mlk_keccak_f1600_x4_native(), respectively, to forward to your + * implementation. + */ + +#if defined(MLK_USE_NATIVE_FIPS202_X1) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 1)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 1)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 1))); +#endif /* MLK_USE_NATIVE_FIPS202_X1 */ +#if defined(MLK_USE_NATIVE_FIPS202_X4) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +__contract__( + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4))); +#endif /* MLK_USE_NATIVE_FIPS202_X4 */ + +/* + * Native x4 XOR bytes and extract bytes interface. + * + * These functions allow backends to provide optimized implementations for + * XORing input data into the state and extracting output data from the state. + * This is particularly useful for backends that use a different internal state + * representation (e.g., bit-interleaved), as conversion can happen during + * XOR/extract rather than before/after each permutation. + * + * NOTE: We assume that the custom representation of the zero state is the + * all-zero state. + * + * MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES: Backend provides native XOR bytes + * MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES: Backend provides native extract + * bytes + */ + +#if defined(MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native( + uint64_t *state, const unsigned char *data0, const unsigned char *data1, + const unsigned char *data2, const unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= 25 * sizeof(uint64_t) && + 0 <= length && length <= 25 * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(memory_no_alias(data0, length)) + requires((data0 == data1 && + data0 == data2 && + data0 == data3) || + (memory_no_alias(data1, length) && + memory_no_alias(data2, length) && + memory_no_alias(data3, length))) + assigns(memory_slice(state, sizeof(uint64_t) * 25 * 4)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged_u64(state, 25 * 4))); +#endif /* MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES */ + +#if defined(MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native( + uint64_t *state, unsigned char *data0, unsigned char *data1, + unsigned char *data2, unsigned char *data3, unsigned offset, + unsigned length) +__contract__( + requires(0 <= offset && offset <= 25 * sizeof(uint64_t) && + 0 <= length && length <= 25 * sizeof(uint64_t) - offset) + requires(memory_no_alias(state, sizeof(uint64_t) * 25 * 4)) + requires(memory_no_alias(data0, length)) + requires(memory_no_alias(data1, length)) + requires(memory_no_alias(data2, length)) + requires(memory_no_alias(data3, length)) + assigns(memory_slice(data0, length)) + assigns(memory_slice(data1, length)) + assigns(memory_slice(data2, length)) + assigns(memory_slice(data3, length)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS)); +#endif /* MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES */ + +#endif /* !MLK_FIPS202_NATIVE_API_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md b/src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md new file mode 100644 index 0000000000..9c2c0d26ea --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/README.md @@ -0,0 +1,10 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# FIPS202 backend for Armv8.1-M + MVE + +This directory contains the source code for a FIPS202 backend targeting +the Armv8.1-M + MVE/Helium architecture. It is automatically derived from +the respective development source in [dev/fips202/armv81m](../../../../../dev/fips202/armv81m). + +**Warning:** This backend is still in active development and has not yet undergone +the same level of review as the rest of the code. Use at your own risk! diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h b/src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h new file mode 100644 index 0000000000..0f87fad5e3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/mve.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_ARMV81M_MVE_H +#define MLK_FIPS202_NATIVE_ARMV81M_MVE_H + +#define MLK_FIPS202_NATIVE_ARMV81M + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 +#define MLK_USE_NATIVE_FIPS202_X4_XOR_BYTES +#define MLK_USE_NATIVE_FIPS202_X4_EXTRACT_BYTES +/* Guard for assembly file */ +#define MLK_FIPS202_ARMV81M_NEED_X4 + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_armv81m.h" + +/* + * Native x4 permutation + * State is kept in bit-interleaved format. + */ +#define mlk_keccak_f1600_x4_native_impl \ + MLK_NAMESPACE(keccak_f1600_x4_native_impl) +int mlk_keccak_f1600_x4_native_impl(uint64_t *state); + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + return mlk_keccak_f1600_x4_native_impl(state); +} + +/* + * Native x4 XOR bytes (with on-the-fly bit interleaving) + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_xor_bytes_x4_native( + uint64_t *state, const uint8_t *data0, const uint8_t *data1, + const uint8_t *data2, const uint8_t *data3, unsigned offset, + unsigned length) +{ + mlk_keccak_f1600_x4_state_xor_bytes_asm(state, data0, data1, data2, data3, + offset, length); + return MLK_NATIVE_FUNC_SUCCESS; +} + +/* + * Native x4 extract bytes (with on-the-fly bit de-interleaving) + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccakf1600_extract_bytes_x4_native( + uint64_t *state, uint8_t *data0, uint8_t *data1, uint8_t *data2, + uint8_t *data3, unsigned offset, unsigned length) +{ + mlk_keccak_f1600_x4_state_extract_bytes_asm(state, data0, data1, data2, data3, + offset, length); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_ARMV81M_MVE_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h new file mode 100644 index 0000000000..7888f337f2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/fips202_native_armv81m.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H +#define MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H + +#include "../../../../common.h" + +/* Keccak round constants in bit-interleaved form */ +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint32_t + mlk_keccakf1600_round_constants[48]; + +#define mlk_keccak_f1600_x4_mve_asm MLK_NAMESPACE(keccak_f1600_x4_mve_asm) +void mlk_keccak_f1600_x4_mve_asm(uint64_t state[100], uint64_t tmpstate[100], + const uint32_t rc[48]); + +#define mlk_keccak_f1600_x4_state_xor_bytes_asm \ + MLK_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm) +void mlk_keccak_f1600_x4_state_xor_bytes_asm(void *state, const uint8_t *d0, + const uint8_t *d1, + const uint8_t *d2, + const uint8_t *d3, unsigned offset, + unsigned length); + +#define mlk_keccak_f1600_x4_state_extract_bytes_asm \ + MLK_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm) +void mlk_keccak_f1600_x4_state_extract_bytes_asm(void *state, uint8_t *data0, + uint8_t *data1, uint8_t *data2, + uint8_t *data3, + unsigned offset, + unsigned length); + +#endif /* !MLK_FIPS202_NATIVE_ARMV81M_SRC_FIPS202_NATIVE_ARMV81M_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S new file mode 100644 index 0000000000..2bf6c3ec18 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.S @@ -0,0 +1,716 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) 2025 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: keccak_f1600_x4_mve_asm + Description: Armv8.1-M MVE implementation of batched (x4) Keccak-f[1600] permutation using bit-interleaved state + Signature: void mlk_keccak_f1600_x4_mve_asm(void *state, void *tmpstate, const uint32_t *rc) + ABI: + Architecture: armv81m + CallingConvention: AAPCS32 + Features: [MVE] + r0: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: void *state + description: Bit-interleaved state for 4 Keccak instances (even halves followed by odd halves) + r1: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: void *tmpstate + description: Temporary storage for intermediate state + r2: + type: buffer + size_bytes: 192 + permissions: read + c_parameter: const uint32_t *rc + description: Keccak round constants in bit-interleaved form (24 pairs of 32-bit words) + Stack: + bytes: 228 + description: register preservation (36) + SIMD registers (64) + temporary storage (128) +*/ + +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/keccak_f1600_x4_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_mve_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_mve_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} + .cfi_adjust_cfa_offset 0x24 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x20 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + sub sp, #0x80 + .cfi_adjust_cfa_offset 0x80 + mov r6, r2 + mov.w lr, #0x18 + mov r2, r0 + mov r4, r1 + add.w r3, r2, #0x190 + vldrw.u32 q0, [r3] + vldrw.u32 q1, [r2] + vldrw.u32 q2, [r2, #32] + wls lr, lr, Lmlk_keccak_f1600_x4_mve_asm_roundend @ imm = #0x8c0 + +Lmlk_keccak_f1600_x4_mve_asm_roundstart: + vldrw.u32 q6, [r2, #112] + veor q7, q6, q2 + vldrw.u32 q2, [r2, #80] + veor q1, q2, q1 + add.w r5, r2, #0x190 + vldrw.u32 q5, [r5, #80] + veor q4, q5, q0 + vldrw.u32 q0, [r2, #192] + veor q3, q7, q0 + vldrw.u32 q0, [r2, #160] + veor q1, q1, q0 + vldrw.u32 q0, [r5, #160] + veor q0, q4, q0 + vldrw.u32 q6, [r2, #272] + veor q2, q3, q6 + vldrw.u32 q7, [r2, #240] + veor q5, q1, q7 + vldrw.u32 q4, [r5, #240] + veor q4, q0, q4 + vldrw.u32 q6, [r2, #352] + veor q3, q2, q6 + vldrw.u32 q0, [r2, #320] + veor q2, q5, q0 + vldrw.u32 q1, [r5, #320] + veor q5, q4, q1 + vldrw.u32 q4, [r5, #32] + veor q0, q3, q5 + vldrw.u32 q1, [r5, #16] + veor q6, q1, q0 + vstrw.32 q5, [sp] + vshr.u32 q7, q6, #0x1f + add.w r10, r4, #0x190 + vsli.32 q7, q6, #0x1 + vldrw.u32 q6, [r5, #112] + veor q6, q4, q6 + vldrw.u32 q4, [r5, #192] + veor q4, q6, q4 + vldrw.u32 q6, [r5, #272] + veor q4, q4, q6 + vldrw.u32 q6, [r5, #352] + veor q5, q4, q6 + vstrw.32 q7, [r4, #160] + vshr.u32 q4, q5, #0x1f + vsli.32 q4, q5, #0x1 + vldrw.u32 q6, [r2, #16] + veor q7, q4, q2 + veor q1, q6, q7 + vldrw.u32 q6, [r5, #96] + veor q6, q6, q0 + vstrw.32 q1, [r10, #160] + vshr.u32 q1, q6, #0xa + vsli.32 q1, q6, #0x16 + vldrw.u32 q6, [r2, #96] + veor q4, q6, q7 + vstrw.32 q1, [r10, #16] + vshr.u32 q6, q4, #0xa + vsli.32 q6, q4, #0x16 + vldrw.u32 q1, [r5, #336] + veor q4, q1, q0 + vldrw.u32 q1, [r2, #176] + veor q1, q1, q7 + vstrw.32 q6, [r4, #16] + vshr.u32 q6, q1, #0x1b + vsli.32 q6, q1, #0x5 + vldrw.u32 q1, [r2, #256] + veor q1, q1, q7 + vstrw.32 q6, [r4, #272] + vshr.u32 q6, q1, #0xa + vsli.32 q6, q1, #0x16 + vldrw.u32 q1, [r2, #336] + veor q1, q1, q7 + vstrw.32 q6, [r10, #128] + vshr.u32 q6, q1, #0x1f + vsli.32 q6, q1, #0x1 + vldrw.u32 q7, [r5, #176] + veor q7, q7, q0 + vstrw.32 q6, [r4, #384] + vshr.u32 q1, q7, #0x1b + vsli.32 q1, q7, #0x5 + vldrw.u32 q6, [r5, #256] + veor q0, q6, q0 + vstrw.32 q1, [r10, #272] + vshr.u32 q1, q4, #0x1f + vldrw.u32 q7, [r5, #64] + vsli.32 q1, q4, #0x1 + vldrw.u32 q4, [r5, #144] + vshr.u32 q6, q0, #0x9 + vstrw.32 q1, [r10, #384] + vsli.32 q6, q0, #0x17 + veor q7, q7, q4 + vldrw.u32 q1, [r5, #224] + veor q4, q7, q1 + vldrw.u32 q7, [r5, #304] + veor q1, q4, q7 + vldrw.u32 q0, [r5, #384] + veor q7, q1, q0 + vstrw.32 q6, [r4, #128] + vshr.u32 q1, q7, #0x1f + vsli.32 q1, q7, #0x1 + vldrw.u32 q6, [r2, #144] + veor q0, q1, q3 + vldrw.u32 q3, [r2, #64] + veor q1, q3, q6 + vldrw.u32 q6, [r2, #224] + veor q1, q1, q6 + vldrw.u32 q3, [r2, #304] + veor q6, q1, q3 + vldrw.u32 q4, [r2, #384] + veor q3, q6, q4 + vldrw.u32 q4, [r2, #48] + veor q5, q3, q5 + vldrw.u32 q1, [r5, #48] + veor q1, q1, q5 + vshr.u32 q6, q1, #0x12 + vsli.32 q6, q1, #0xe + vldrw.u32 q1, [r2, #128] + veor q1, q1, q0 + vstrw.32 q6, [r10, #80] + vshr.u32 q6, q1, #0x5 + vsli.32 q6, q1, #0x1b + vldrw.u32 q1, [r5, #128] + veor q1, q1, q5 + vstrw.32 q6, [r10, #336] + vshr.u32 q6, q1, #0x4 + vsli.32 q6, q1, #0x1c + veor q1, q4, q0 + vstrw.32 q6, [r4, #336] + vshr.u32 q4, q1, #0x12 + vsli.32 q4, q1, #0xe + vldrw.u32 q6, [r2, #208] + veor q6, q6, q0 + vstrw.32 q4, [r4, #80] + vshr.u32 q1, q6, #0x14 + vsli.32 q1, q6, #0xc + vldrw.u32 q4, [r2, #288] + veor q4, q4, q0 + vldrw.u32 q6, [r2, #368] + veor q0, q6, q0 + vshr.u32 q6, q0, #0x4 + vstrw.32 q1, [r10, #192] + vsli.32 q6, q0, #0x1c + vshr.u32 q0, q4, #0x16 + vldrw.u32 q1, [r5, #368] + vsli.32 q0, q4, #0xa + vstrw.32 q6, [r4, #304] + veor q4, q1, q5 + vstrw.32 q0, [r10, #48] + vshr.u32 q1, q4, #0x4 + vsli.32 q1, q4, #0x1c + vldrw.u32 q6, [r5, #208] + veor q6, q6, q5 + vldrw.u32 q0, [r5, #288] + veor q5, q0, q5 + vstrw.32 q1, [r10, #304] + vshr.u32 q0, q6, #0x13 + vsli.32 q0, q6, #0xd + vldrw.u32 q1, [r5, #96] + vshr.u32 q6, q5, #0x15 + vldrw.u32 q4, [r5, #16] + vsli.32 q6, q5, #0xb + vldrw.u32 q5, [r5, #176] + veor q1, q4, q1 + vldrw.u32 q4, [r5, #256] + veor q5, q1, q5 + vldrw.u32 q1, [r5, #336] + veor q5, q5, q4 + vstrw.32 q0, [r4, #192] + veor q0, q5, q1 + vstrw.32 q6, [r4, #48] + vshr.u32 q5, q0, #0x1f + vsli.32 q5, q0, #0x1 + vldrw.u32 q4, [r2, #16] + veor q3, q5, q3 + vldrw.u32 q6, [r2, #96] + veor q4, q4, q6 + vldrw.u32 q1, [r2, #176] + veor q5, q4, q1 + vldrw.u32 q6, [r2, #256] + veor q6, q5, q6 + vldrw.u32 q4, [r2, #336] + veor q5, q6, q4 + vldrw.u32 q1, [r5] + veor q7, q5, q7 + vldrw.u32 q4, [r2] + veor q1, q1, q7 + veor q4, q4, q3 + vshr.u32 q6, q1, #0x20 + vsli.32 q6, q1, #0x0 + vldrw.u32 q1, [r2, #80] + veor q1, q1, q3 + vstrw.32 q6, [r10] + vshr.u32 q6, q4, #0x20 + vsli.32 q6, q4, #0x0 + vldrw.u32 q4, [r5, #80] + veor q4, q4, q7 + vstrw.32 q6, [r4] + vshr.u32 q6, q1, #0xe + vsli.32 q6, q1, #0x12 + vldrw.u32 q1, [r2, #160] + veor q1, q1, q3 + vstrw.32 q6, [r4, #256] + vshr.u32 q6, q4, #0xe + vsli.32 q6, q4, #0x12 + vldrw.u32 q4, [r2, #240] + veor q4, q4, q3 + vstrw.32 q6, [r10, #256] + vshr.u32 q6, q1, #0x1f + vsli.32 q6, q1, #0x1 + vldrw.u32 q1, [r2, #320] + veor q1, q1, q3 + vstrw.32 q6, [r10, #112] + vshr.u32 q6, q4, #0xc + vsli.32 q6, q4, #0x14 + vldrw.u32 q3, [r5, #240] + veor q3, q3, q7 + vstrw.32 q6, [r10, #368] + vshr.u32 q4, q3, #0xb + vsli.32 q4, q3, #0x15 + vldrw.u32 q3, [r5, #160] + veor q6, q3, q7 + vstrw.32 q4, [r4, #368] + vshr.u32 q3, q6, #0x1e + vsli.32 q3, q6, #0x2 + vldrw.u32 q6, [r5, #320] + veor q7, q6, q7 + vldrw.u32 q4, [r2, #368] + vshr.u32 q6, q1, #0x17 + vstrw.32 q3, [r4, #112] + vsli.32 q6, q1, #0x9 + vshr.u32 q1, q7, #0x17 + vldrw.u32 q3, [r2, #48] + vsli.32 q1, q7, #0x9 + vldrw.u32 q7, [r2, #128] + veor q3, q3, q7 + vldrw.u32 q7, [r2, #208] + veor q7, q3, q7 + vldrw.u32 q3, [r2, #288] + veor q3, q7, q3 + vldrw.u32 q7, [r5, #128] + veor q3, q3, q4 + vldrw.u32 q4, [r5, #48] + veor q0, q3, q0 + veor q4, q4, q7 + vldrw.u32 q7, [r5, #208] + veor q4, q4, q7 + vldrw.u32 q7, [r5, #288] + veor q4, q4, q7 + vldrw.u32 q7, [r5, #368] + veor q7, q4, q7 + vstrw.32 q6, [r4, #224] + vshr.u32 q4, q7, #0x1f + vstrw.32 q1, [r10, #224] + vsli.32 q4, q7, #0x1 + veor q5, q4, q5 + vldrw.u32 q6, [r2, #192] + veor q1, q6, q5 + vldrw.u32 q4, [r5, #112] + veor q7, q2, q7 + vldrw.u32 q6, [r5, #32] + vshr.u32 q2, q1, #0xb + vsli.32 q2, q1, #0x15 + veor q1, q6, q0 + vstrw.32 q2, [r10, #32] + vshr.u32 q6, q1, #0x1 + vsli.32 q6, q1, #0x1f + vldrw.u32 q2, [r2, #112] + veor q2, q2, q5 + vstrw.32 q6, [r10, #320] + vshr.u32 q1, q2, #0x1d + vsli.32 q1, q2, #0x3 + vldrw.u32 q6, [r2, #32] + veor q4, q4, q0 + vstrw.32 q1, [r4, #176] + veor q2, q6, q5 + vshr.u32 q6, q2, #0x1 + vldrw.u32 q1, [r5, #352] + vsli.32 q6, q2, #0x1f + veor q1, q1, q0 + vstrw.32 q6, [r4, #320] + vshr.u32 q6, q1, #0x1 + vsli.32 q6, q1, #0x1f + vldrw.u32 q2, [r5, #192] + vshr.u32 q1, q4, #0x1d + vstrw.32 q6, [r4, #144] + vsli.32 q1, q4, #0x3 + veor q2, q2, q0 + vldrw.u32 q6, [r5, #272] + veor q0, q6, q0 + vldrw.u32 q4, [r2, #352] + veor q6, q4, q5 + vldrw.u32 q4, [r2, #272] + veor q4, q4, q5 + vstrw.32 q1, [r10, #176] + vshr.u32 q1, q2, #0xa + vsli.32 q1, q2, #0x16 + vldrw.u32 q5, [sp] + vshr.u32 q2, q0, #0x18 + vstrw.32 q1, [r4, #32] + vsli.32 q2, q0, #0x8 + vshr.u32 q1, q6, #0x2 + vstrw.32 q2, [r4, #288] + vsli.32 q1, q6, #0x1e + vshr.u32 q6, q4, #0x19 + vstrw.32 q1, [r10, #144] + vsli.32 q6, q4, #0x7 + vshr.u32 q0, q5, #0x1f + vstrw.32 q6, [r10, #288] + vsli.32 q0, q5, #0x1 + veor q5, q0, q3 + vldrw.u32 q6, [r2, #64] + veor q3, q6, q5 + vldrw.u32 q1, [r5, #64] + vshr.u32 q4, q3, #0x13 + vldrw.u32 q2, [r2, #384] + vsli.32 q4, q3, #0xd + vldrw.u32 q0, [r5, #224] + veor q6, q1, q7 + vstrw.32 q4, [r10, #240] + veor q2, q2, q5 + veor q3, q0, q7 + vldrw.u32 q0, [r2, #224] + vshr.u32 q4, q6, #0x12 + vldrw.u32 q1, [r5, #384] + vsli.32 q4, q6, #0xe + vshr.u32 q6, q2, #0x19 + vstrw.32 q4, [r4, #240] + vsli.32 q6, q2, #0x7 + vshr.u32 q2, q3, #0xc + vstrw.32 q6, [r4, #64] + vsli.32 q2, q3, #0x14 + veor q0, q0, q5 + vldrw.u32 q6, [r2, #144] + veor q4, q1, q7 + veor q6, q6, q5 + vstrw.32 q2, [r4, #352] + vshr.u32 q2, q4, #0x19 + vsli.32 q2, q4, #0x7 + vldrw.u32 q1, [r2, #304] + veor q5, q1, q5 + vldrw.u32 q1, [r5, #144] + veor q4, q1, q7 + vldrw.u32 q3, [r5, #304] + veor q1, q3, q7 + vstrw.32 q2, [r10, #64] + vshr.u32 q3, q0, #0xd + vsli.32 q3, q0, #0x13 + vldrw.u32 q7, [r4, #80] + vshr.u32 q0, q6, #0x16 + vstrw.32 q3, [r10, #352] + vsli.32 q0, q6, #0xa + vshr.u32 q2, q5, #0x1c + vsli.32 q2, q5, #0x4 + vldrw.u32 q5, [r4, #112] + vshr.u32 q3, q1, #0x1c + vsli.32 q3, q1, #0x4 + vldrw.u32 q1, [r4, #128] + vbic q6, q5, q0 + vstrw.32 q3, [r10, #208] + vbic q3, q1, q5 + veor q3, q0, q3 + vstrw.32 q3, [r2, #96] + vbic q3, q0, q7 + veor q0, q7, q6 + vldrw.u32 q6, [r4, #144] + vbic q7, q7, q6 + vstrw.32 q0, [r2, #80] + veor q3, q6, q3 + vstrw.32 q3, [r2, #144] + veor q0, q1, q7 + vstrw.32 q0, [r2, #128] + vbic q1, q6, q1 + vshr.u32 q6, q4, #0x16 + vldrw.u32 q3, [r10, #112] + vsli.32 q6, q4, #0xa + vldrw.u32 q4, [r10, #80] + veor q1, q5, q1 + vldrw.u32 q0, [r10, #144] + vbic q7, q4, q0 + vldrw.u32 q5, [r10, #128] + veor q7, q5, q7 + vstrw.32 q1, [r2, #112] + vbic q1, q0, q5 + vstrw.32 q7, [r5, #128] + veor q7, q3, q1 + vstrw.32 q7, [r5, #112] + vbic q7, q5, q3 + vbic q1, q3, q6 + vldrw.u32 q3, [r4, #176] + veor q5, q4, q1 + vbic q4, q6, q4 + vldrw.u32 q1, [r4, #160] + veor q0, q0, q4 + vldrw.u32 q4, [r4, #224] + veor q7, q6, q7 + vstrw.32 q0, [r5, #144] + vbic q0, q1, q4 + vstrw.32 q7, [r5, #96] + veor q0, q2, q0 + vstrw.32 q0, [r2, #208] + vbic q6, q3, q1 + vstrw.32 q5, [r5, #80] + vbic q7, q4, q2 + vldrw.u32 q0, [r10, #160] + veor q6, q4, q6 + vldrw.u32 q5, [r4, #192] + vbic q4, q2, q5 + vldrw.u32 q2, [r10, #224] + veor q4, q3, q4 + vstrw.32 q4, [r2, #176] + vbic q4, q5, q3 + vstrw.32 q6, [r2, #224] + veor q4, q1, q4 + vldrw.u32 q1, [r10, #208] + veor q3, q5, q7 + vldrw.u32 q5, [r10, #192] + vbic q6, q1, q5 + vldrw.u32 q7, [r10, #176] + veor q6, q7, q6 + vstrw.32 q3, [r2, #192] + vbic q3, q0, q2 + vstrw.32 q6, [r5, #176] + veor q3, q1, q3 + vstrw.32 q3, [r5, #208] + vbic q3, q5, q7 + vstrw.32 q4, [r2, #160] + veor q3, q0, q3 + vstrw.32 q3, [r5, #160] + vbic q6, q2, q1 + vldrw.u32 q1, [r4, #288] + vbic q7, q7, q0 + vldrw.u32 q3, [r4, #272] + veor q0, q5, q6 + vldrw.u32 q4, [r4, #304] + veor q6, q2, q7 + vldrw.u32 q7, [r4, #256] + vbic q5, q4, q1 + vstrw.32 q0, [r5, #192] + veor q5, q3, q5 + vstrw.32 q6, [r5, #224] + vbic q0, q3, q7 + vstrw.32 q5, [r2, #272] + vbic q6, q1, q3 + veor q5, q7, q6 + vldrw.u32 q3, [r4, #240] + veor q6, q3, q0 + vldrw.u32 q2, [r10, #288] + vbic q0, q3, q4 + vstrw.32 q6, [r2, #240] + vbic q7, q7, q3 + vstrw.32 q5, [r2, #256] + veor q7, q4, q7 + vstrw.32 q7, [r2, #304] + veor q7, q1, q0 + vstrw.32 q7, [r2, #288] + vldrw.u32 q5, [r10, #304] + vbic q7, q5, q2 + vldrw.u32 q3, [r10, #272] + veor q1, q3, q7 + vldrw.u32 q7, [r4, #336] + vbic q4, q2, q3 + vldrw.u32 q6, [r10, #256] + vbic q3, q3, q6 + vldrw.u32 q0, [r10, #240] + veor q3, q0, q3 + vstrw.32 q1, [r5, #272] + vbic q1, q0, q5 + vstrw.32 q3, [r5, #240] + veor q1, q2, q1 + vldrw.u32 q3, [r4, #384] + vbic q2, q6, q0 + vldrw.u32 q0, [r4, #320] + veor q2, q5, q2 + vldrw.u32 q5, [r4, #352] + veor q4, q6, q4 + vstrw.32 q2, [r5, #304] + vbic q2, q7, q0 + vstrw.32 q1, [r5, #288] + veor q1, q3, q2 + vstrw.32 q1, [r2, #384] + vbic q2, q5, q7 + vstrw.32 q4, [r5, #256] + veor q4, q0, q2 + vstrw.32 q4, [r2, #320] + vbic q2, q0, q3 + vldrw.u32 q4, [r4, #368] + vbic q3, q3, q4 + vldrw.u32 q0, [r10, #320] + veor q1, q5, q3 + vldrw.u32 q6, [r10, #336] + vbic q5, q4, q5 + vstrw.32 q1, [r2, #352] + veor q5, q7, q5 + vstrw.32 q5, [r2, #336] + veor q3, q4, q2 + vstrw.32 q3, [r2, #368] + vbic q7, q6, q0 + vldrw.u32 q5, [r10, #352] + vbic q3, q5, q6 + vldrw.u32 q1, [r10, #368] + vbic q4, q1, q5 + vldrw.u32 q2, [r4, #16] + veor q6, q6, q4 + vldrw.u32 q4, [r10, #384] + veor q3, q0, q3 + vstrw.32 q3, [r5, #320] + veor q3, q4, q7 + vstrw.32 q3, [r5, #384] + vbic q0, q0, q4 + vstrw.32 q6, [r5, #336] + veor q3, q1, q0 + vstrw.32 q3, [r5, #368] + vbic q7, q4, q1 + veor q5, q5, q7 + vldrw.u32 q6, [r4, #32] + vbic q3, q6, q2 + vldrw.u32 q4, [r4, #48] + vbic q0, q4, q6 + vldrw.u32 q1, [r4] + veor q0, q2, q0 + vldrw.u32 q7, [r4, #64] + veor q3, q1, q3 + vstrw.32 q5, [r5, #352] + vbic q5, q1, q7 + vstrw.32 q0, [r2, #16] + veor q0, q4, q5 + vstrw.32 q0, [r2, #48] + vbic q5, q2, q1 + veor q2, q7, q5 + vldrw.u32 q0, [r10, #16] + vbic q5, q7, q4 + vldrw.u32 q4, [r10] + vbic q1, q0, q4 + vldrw.u32 q7, [r10, #64] + veor q1, q7, q1 + vstrw.32 q2, [r2, #64] + veor q2, q6, q5 + vbic q6, q4, q7 + vldrw.u32 q5, [r10, #48] + veor q6, q5, q6 + ldrd r7, r8, [r6] + vbic q7, q7, q5 + vstrw.32 q1, [r5, #64] + vdup.32 q1, r7 + veor q1, q3, q1 + vldrw.u32 q3, [r10, #32] + veor q7, q3, q7 + add.w r6, r6, #0x8 + vbic q5, q5, q3 + vstrw.32 q6, [r5, #48] + vbic q6, q3, q0 + vstrw.32 q1, [r2] + veor q5, q0, q5 + vstrw.32 q7, [r5, #32] + veor q4, q4, q6 + vstrw.32 q5, [r5, #16] + vdup.32 q6, r8 + vstrw.32 q2, [r2, #32] + veor q0, q4, q6 + vstrw.32 q0, [r5] + +Lmlk_keccak_f1600_x4_mve_asm_roundend_pre: + le lr, Lmlk_keccak_f1600_x4_mve_asm_roundstart @ imm = #-0x8c0 + +Lmlk_keccak_f1600_x4_mve_asm_roundend: + add sp, #0x80 + .cfi_adjust_cfa_offset -0x80 + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x24 + .cfi_endproc + nop + +MLK_ASM_FN_SIZE(keccak_f1600_x4_mve_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c new file mode 100644 index 0000000000..d0c06b5324 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../../common.h" +#include "../../../../verify.h" + +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_armv81m.h" + + +/* + * Keccak-f1600 x4 permutation (on bit-interleaved state) + * State is expected to already be in bit-interleaved format. + */ +#define mlk_keccak_f1600_x4_native_impl \ + MLK_NAMESPACE(keccak_f1600_x4_native_impl) +int mlk_keccak_f1600_x4_native_impl(uint64_t *state) +{ + MLK_ALIGN uint64_t state_tmp[100]; + mlk_keccak_f1600_x4_mve_asm(state, state_tmp, + mlk_keccakf1600_round_constants); + mlk_zeroize(state_tmp, sizeof(state_tmp)); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(keccak_f1600_x4_mve) + +#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \ + */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +/* Some macros are kept because they are also defined in a header. */ +/* Keep: mlk_keccak_f1600_x4_native_impl (mve.h) */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S new file mode 100644 index 0000000000..43aaa95c53 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S @@ -0,0 +1,334 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2026 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// --------------------------------------------------------------------------- +// Overview +// --------------------------------------------------------------------------- +// MVE/Helium implementation of KeccakF1600x4_StateExtractBytes +// (inverse of keccak_f1600_x4_state_xor_bytes_mve.S). +// +// void KeccakF1600x4_StateExtractBytes(state, d0, d1, d2, d3, offset, length) +// +// Reads 'length' bytes from the bit-interleaved Keccak state starting at +// byte 'offset', recombines the even and odd halves of each lane back +// into plain bytes, and writes them to four output buffers (d0..d3). +// +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd +// +// --------------------------------------------------------------------------- +// Three-phase structure +// --------------------------------------------------------------------------- +// Prologue -- if offset is not 8-byte aligned, extract +// min(length, 8-(offset%8)) bytes via predicated byte stores. +// Main -- process full 8-byte groups: load even/odd lane pair, +// de-interleave, scatter-store to output buffers. +// Tail -- extract remaining <8 bytes via predicated byte stores. + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/keccak_f1600_x4_state_extract_bytes_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_extract_bytes_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr} + .cfi_adjust_cfa_offset 0x28 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x24 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldr r4, [sp, #0x68] + ldr.w r10, [sp, #0x6c] + ldr r6, [sp, #0x70] + cmp r6, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x2ea + and r5, r10, #0x7 + bic r9, r10, #0x7 + add.w r8, r0, r9, lsl #1 + add.w r7, r8, #0x190 + cmp r5, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_extract_bytes_asm_pre_main @ imm = #0x112 + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov.f64 d4, d1 + vmov.f64 d6, d3 + rsb.w lr, r5, #0x8 + cmp r6, lr + it ls + movls lr, r6 + vctp.8 lr + vmrs r11, p0 + lsl.w r11, r11, r5 + vmsr p0, r11 + subs r1, r1, r5 + subs r2, r2, r5 + subs r3, r3, r5 + subs r4, r4, r5 + vpstttt + vstrbt.8 q0, [r1], #4 + vstrbt.8 q1, [r2], #4 + vstrbt.8 q2, [r3], #4 + vstrbt.8 q3, [r4], #4 + subs.w r6, r6, lr + cmp r6, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x1cc + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + b Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_body @ imm = #0xe + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_pre_main: + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + mov.w r12, #0x4 + vsub.i32 q7, q7, r12 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_body: + lsr.w lr, r6, #0x3 + wls lr, lr, Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_end @ imm = #0xb4 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_start: + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vstrw.32 q0, [q7, #4]! + vstrw.32 q1, [q7, #4]! + le lr, Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_start @ imm = #-0xb4 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_main_loop_end: + ands r6, r6, #0x7 + beq Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0xee + mov.w r12, #0x4 + vadd.i32 q7, q7, r12 + vmov r1, r3, q7[2], q7[0] + vmov r2, r4, q7[3], q7[1] + vldrw.u32 q0, [r8], #16 + vldrw.u32 q1, [r7], #16 + vrev32.16 q2, q0 + vrev32.16 q3, q1 + vsli.32 q0, q0, #0x8 + vsli.16 q0, q0, #0x4 + vsli.8 q0, q0, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x4 + vshr.u8 q4, q0, #0x5 + vsli.8 q0, q4, #0x6 + vsli.32 q1, q1, #0x8 + vsli.16 q1, q1, #0x4 + vsli.8 q1, q1, #0x1 + vshr.u8 q4, q1, #0x3 + vsli.8 q1, q4, #0x4 + vshr.u8 q4, q1, #0x5 + vsli.8 q1, q4, #0x6 + mov.w r0, #0x55 + vdup.8 q4, r0 + vand q0, q0, q4 + vand q1, q1, q4 + vshl.i32 q1, q1, #0x1 + vorr q0, q0, q1 + vsli.32 q2, q2, #0x8 + vsli.16 q2, q2, #0x4 + vsli.8 q2, q2, #0x1 + vshr.u8 q1, q2, #0x3 + vsli.8 q2, q1, #0x4 + vshr.u8 q1, q2, #0x5 + vsli.8 q2, q1, #0x6 + vsli.32 q3, q3, #0x8 + vsli.16 q3, q3, #0x4 + vsli.8 q3, q3, #0x1 + vshr.u8 q1, q3, #0x3 + vsli.8 q3, q1, #0x4 + vshr.u8 q1, q3, #0x5 + vsli.8 q3, q1, #0x6 + vand q1, q2, q4 + vand q3, q3, q4 + vshl.i32 q3, q3, #0x1 + vorr q1, q1, q3 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov.f64 d4, d1 + vmov.f64 d6, d3 + vctp.8 r6 + vpstttt + vstrbt.8 q0, [r1], #4 + vstrbt.8 q1, [r2], #4 + vstrbt.8 q2, [r3], #4 + vstrbt.8 q3, [r4], #4 + +Lmlk_keccak_f1600_x4_state_extract_bytes_asm_exit: + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x28 + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_state_extract_bytes_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S new file mode 100644 index 0000000000..fd58e7cbe6 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S @@ -0,0 +1,355 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * Copyright (c) 2026 Arm Limited + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +// --------------------------------------------------------------------------- +// Overview +// --------------------------------------------------------------------------- +// MVE/Helium implementation of KeccakF1600x4_StateXORBytes. +// +// void KeccakF1600x4_StateXORBytes(state, d0, d1, d2, d3, offset, length) +// +// Reads 'length' plain bytes from each of four input buffers (d0..d3), +// splits every byte into its even and odd bits (bit-interleaving), and +// XORs the result into the Keccak state starting at byte 'offset'. +// +// --------------------------------------------------------------------------- +// Bit-interleaving background +// --------------------------------------------------------------------------- +// Each 64-bit Keccak lane is stored as two 32-bit words: +// even half -- bits 0, 2, 4, ..., 62 of the lane +// odd half -- bits 1, 3, 5, ..., 63 of the lane +// This representation allows 64-bit lane rotations (used in the Keccak +// round function) to be implemented as pairs of 32-bit rotations. +// +// Batched (x4) processing: +// Four Keccak instances are processed as a batch. Their states are +// stored interleaved in a single 800-byte buffer: first the even +// halves of all 25 lanes (400 bytes), then the odd halves (400 bytes). +// Within each 16-byte row, the four u32 words correspond to +// instances 0..3 of the same lane, enabling SIMD-parallel operations +// across all four instances. +// +// State memory layout (25 lanes x 4 instances x 2 halves): +// S[i][l]_even/odd = even/odd half of lane l, instance i (u32) +// Each row is 16 bytes (one Q-register). +// Offset Contents +// 0 S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even +// 16 S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even +// ... +// 384 S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even +// 400 S[0][ 0]_odd, S[1][ 0]_odd, S[2][ 0]_odd, S[3][ 0]_odd +// 416 S[0][ 1]_odd, S[1][ 1]_odd, S[2][ 1]_odd, S[3][ 1]_odd +// ... +// 784 S[0][24]_odd, S[1][24]_odd, S[2][24]_odd, S[3][24]_odd +// +// --------------------------------------------------------------------------- +// Three-phase structure +// --------------------------------------------------------------------------- +// Prologue -- if offset is not 8-byte aligned, absorb +// min(length, 8-(offset%8)) bytes via predicated byte loads. +// Main -- process full 8-byte groups via word-level gather loads, +// bit-interleave, then VEOR into even/odd state halves. +// Tail -- absorb remaining <8 bytes via predicated byte loads. + +#include "../../../../common.h" +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/armv81m/src/keccak_f1600_x4_state_xor_bytes_mve.S using scripts/simpasm. Do not modify it directly. + */ + +.thumb +.syntax unified + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_state_xor_bytes_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) + + .cfi_startproc + push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr} + .cfi_adjust_cfa_offset 0x28 + .cfi_rel_offset r4, 0x0 + .cfi_rel_offset r5, 0x4 + .cfi_rel_offset r6, 0x8 + .cfi_rel_offset r7, 0xc + .cfi_rel_offset r8, 0x10 + .cfi_rel_offset r9, 0x14 + .cfi_rel_offset r10, 0x18 + .cfi_rel_offset r11, 0x1c + .cfi_rel_offset lr, 0x24 + vpush {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_adjust_cfa_offset 0x40 + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + ldr r4, [sp, #0x68] + ldr.w r10, [sp, #0x6c] + ldr r6, [sp, #0x70] + cmp r6, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x346 + and r5, r10, #0x7 + bic r9, r10, #0x7 + add.w r8, r0, r9, lsl #1 + add.w r7, r8, #0x190 + cmp r5, #0x0 + beq.w Lmlk_keccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x12c + subs r1, r1, r5 + subs r2, r2, r5 + subs r3, r3, r5 + subs r4, r4, r5 + rsb.w lr, r5, #0x8 + cmp r6, lr + it ls + movls lr, r6 + subs.w r6, r6, lr + vctp.8 lr + vmrs r11, p0 + lsl.w r11, r11, r5 + vmsr p0, r11 + vpstttt + vldrbt.u8 q0, [r1], #4 + vldrbt.u8 q1, [r2], #4 + vldrbt.u8 q2, [r3], #4 + vldrbt.u8 q3, [r4], #4 + vmov.f64 d1, d4 + vmov.f64 d3, d6 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + b Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_body @ imm = #0xe + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_pre_main: + vmov q7[2], q7[0], r1, r3 + vmov q7[3], q7[1], r2, r4 + mov.w r0, #0x4 + vsub.i32 q7, q7, r0 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_body: + lsr.w lr, r6, #0x3 + wls lr, lr, Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_end @ imm = #0xd4 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_start: + vldrw.u32 q0, [q7, #4]! + vldrw.u32 q1, [q7, #4]! + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + le lr, Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_start @ imm = #-0xd4 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_main_loop_end: + ands r6, r6, #0x7 + beq.w Lmlk_keccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x110 + mov.w r0, #0x4 + vadd.i32 q7, q7, r0 + vmov r1, r3, q7[2], q7[0] + vmov r2, r4, q7[3], q7[1] + vctp.8 r6 + vpstttt + vldrbt.u8 q0, [r1] + vldrbt.u8 q1, [r2] + vldrbt.u8 q2, [r3] + vldrbt.u8 q3, [r4] + vmov.f64 d1, d4 + vmov.f64 d3, d6 + vrev64.32 q2, q0 + vrev64.32 q3, q1 + movw r0, #0xf0f + vmsr p0, r0 + vpsel q0, q0, q3 + vpsel q1, q2, q1 + vmov q2, q0 + vmov q3, q1 + vshr.u8 q4, q0, #0x2 + vsli.8 q0, q4, #0x1 + vshr.u8 q4, q0, #0x3 + vsli.8 q0, q4, #0x2 + vshr.u8 q4, q0, #0x4 + vsli.8 q0, q4, #0x3 + vshr.u16 q4, q0, #0x8 + vsli.8 q0, q4, #0x4 + vshr.u32 q4, q0, #0x10 + vsli.16 q0, q4, #0x8 + vshr.u8 q4, q3, #0x2 + vsli.8 q3, q4, #0x1 + vshr.u8 q4, q3, #0x3 + vsli.8 q3, q4, #0x2 + vshr.u8 q4, q3, #0x4 + vsli.8 q3, q4, #0x3 + vshr.u16 q4, q3, #0x8 + vsli.8 q3, q4, #0x4 + vshr.u32 q4, q3, #0x10 + vsli.16 q3, q4, #0x8 + vsli.32 q0, q3, #0x10 + vshl.i8 q4, q2, #0x2 + vsri.8 q2, q4, #0x1 + vshl.i8 q4, q2, #0x3 + vsri.8 q2, q4, #0x2 + vshl.i8 q4, q2, #0x4 + vsri.8 q2, q4, #0x3 + vshl.i16 q4, q2, #0x8 + vsri.8 q2, q4, #0x4 + vshl.i32 q4, q2, #0x10 + vsri.16 q2, q4, #0x8 + vshl.i8 q4, q1, #0x2 + vsri.8 q1, q4, #0x1 + vshl.i8 q4, q1, #0x3 + vsri.8 q1, q4, #0x2 + vshl.i8 q4, q1, #0x4 + vsri.8 q1, q4, #0x3 + vshl.i16 q4, q1, #0x8 + vsri.8 q1, q4, #0x4 + vshl.i32 q4, q1, #0x10 + vsri.16 q1, q4, #0x8 + vsri.32 q1, q2, #0x10 + vldrw.u32 q4, [r8] + vldrw.u32 q5, [r7] + veor q4, q4, q0 + veor q5, q5, q1 + vstrw.32 q4, [r8], #16 + vstrw.32 q5, [r7], #16 + +Lmlk_keccak_f1600_x4_state_xor_bytes_asm_exit: + vpop {d8, d9, d10, d11, d12, d13, d14, d15} + .cfi_restore d8 + .cfi_restore d9 + .cfi_restore d10 + .cfi_restore d11 + .cfi_restore d12 + .cfi_restore d13 + .cfi_restore d14 + .cfi_restore d15 + .cfi_adjust_cfa_offset -0x40 + pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc} + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 + .cfi_restore r10 + .cfi_restore r11 + .cfi_restore lr + .cfi_adjust_cfa_offset -0x28 + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_state_xor_bytes_asm) + +#endif /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c new file mode 100644 index 0000000000..b19bf33271 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/armv81m/src/keccakf1600_round_constants.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../../common.h" + +#if defined(MLK_FIPS202_ARMV81M_NEED_X4) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "fips202_native_armv81m.h" + +/* + * Keccak round constants in bit-interleaved form. + * Each 64-bit constant is split into two 32-bit words: + * - low word contains even-indexed bits + * - high word contains odd-indexed bits + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint32_t + mlk_keccakf1600_round_constants[48] = { + 0x00000001, 0x00000000, /* RC0 */ + 0x00000000, 0x00000089, /* RC1 */ + 0x00000000, 0x8000008b, /* RC2 */ + 0x00000000, 0x80008080, /* RC3 */ + 0x00000001, 0x0000008b, /* RC4 */ + 0x00000001, 0x00008000, /* RC5 */ + 0x00000001, 0x80008088, /* RC6 */ + 0x00000001, 0x80000082, /* RC7 */ + 0x00000000, 0x0000000b, /* RC8 */ + 0x00000000, 0x0000000a, /* RC9 */ + 0x00000001, 0x00008082, /* RC10 */ + 0x00000000, 0x00008003, /* RC11 */ + 0x00000001, 0x0000808b, /* RC12 */ + 0x00000001, 0x8000000b, /* RC13 */ + 0x00000001, 0x8000008a, /* RC14 */ + 0x00000001, 0x80000081, /* RC15 */ + 0x00000000, 0x80000081, /* RC16 */ + 0x00000000, 0x80000008, /* RC17 */ + 0x00000000, 0x00000083, /* RC18 */ + 0x00000000, 0x80008003, /* RC19 */ + 0x00000001, 0x80008088, /* RC20 */ + 0x00000000, 0x80000088, /* RC21 */ + 0x00000001, 0x00008000, /* RC22 */ + 0x00000000, 0x80008082, /* RC23 */ +}; + +#else /* MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(fips202_armv81m_round_constants) + +#endif /* !(MLK_FIPS202_ARMV81M_NEED_X4 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) \ + */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/auto.h b/src/mlkem-native/mlkem/src/fips202/native/auto.h new file mode 100644 index 0000000000..e5a6819402 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/auto.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_AUTO_H +#define MLK_FIPS202_NATIVE_AUTO_H + +/* + * Default FIPS202 backend + */ +#include "../../sys.h" + +#if defined(MLK_SYS_AARCH64) +#include "aarch64/auto.h" +#endif + +/* The x86_64 backend requires toolchain support for the SysV ABI */ +#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED) +#include "x86_64/keccak_f1600_x4_avx2.h" +#endif + +/* We do not yet include the FIPS202 backend for Armv8.1-M+MVE by default + * as it is still experimental and undergoing review. */ +/* #if defined(MLK_SYS_ARMV81M_MVE) */ +/* #include "armv81m/mve.h" */ +/* #endif */ + +#endif /* !MLK_FIPS202_NATIVE_AUTO_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h b/src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h new file mode 100644 index 0000000000..0a16b3782d --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H +#define MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H + +#include "../../../common.h" + +#define MLK_FIPS202_X86_64_NEED_X4_AVX2 + +/* Part of backend API */ +#define MLK_USE_NATIVE_FIPS202_X4 + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/fips202_native_x86_64.h" +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x4_native(uint64_t *state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x4_avx2_asm(state, mlk_keccakf1600_round_constants, + mlk_keccak_rho8, mlk_keccak_rho56); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_FIPS202_NATIVE_X86_64_KECCAK_F1600_X4_AVX2_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h new file mode 100644 index 0000000000..69ce1749ac --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/fips202_native_x86_64.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H +#define MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H + +#include "../../../../cbmc.h" +#include "../../../../common.h" + +/* TODO: Reconsider whether this check is needed -- x86_64 is always + * little-endian, so the backend selection already implies this. */ +#ifndef MLK_SYS_LITTLE_ENDIAN +#error Expecting a little-endian platform +#endif + +#define mlk_keccakf1600_round_constants \ + MLK_NAMESPACE(keccakf1600_round_constants) +MLK_INTERNAL_DATA_DECLARATION const uint64_t + mlk_keccakf1600_round_constants[24]; + +#define mlk_keccak_rho8 MLK_NAMESPACE(keccak_rho8) +MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho8[4]; + +#define mlk_keccak_rho56 MLK_NAMESPACE(keccak_rho56) +MLK_INTERNAL_DATA_DECLARATION const uint64_t mlk_keccak_rho56[4]; + +#define mlk_keccak_f1600_x4_avx2_asm MLK_NAMESPACE(keccak_f1600_x4_avx2_asm) +MLK_SYSV_ABI +void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24], + const uint64_t rho8[4], + const uint64_t rho56[4]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/keccak_f1600_x4_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(states, sizeof(uint64_t) * 25 * 4)) + requires(rc == mlk_keccakf1600_round_constants) + requires(rho8 == mlk_keccak_rho8) + requires(rho56 == mlk_keccak_rho56) + assigns(memory_slice(states, sizeof(uint64_t) * 25 * 4)) +); + +#endif /* !MLK_FIPS202_NATIVE_X86_64_SRC_FIPS202_NATIVE_X86_64_H */ diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S new file mode 100644 index 0000000000..c449ed279b --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccak_f1600_x4_avx2_asm.S @@ -0,0 +1,487 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#include "../../../../common.h" + +#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: keccak_f1600_x4_avx2_asm + Description: x86_64 AVX2 Keccak-f[1600] permutation for four sequential states + Signature: void mlk_keccak_f1600_x4_avx2_asm(uint64_t states[100], const uint64_t rc[24], const uint64_t rho8[4], const uint64_t rho56[4]) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 800 + permissions: read/write + c_parameter: uint64_t states[100] + description: Four sequential Keccak states (4 x 25 x uint64_t) + rsi: + type: buffer + size_bytes: 192 + permissions: read-only + c_parameter: const uint64_t rc[24] + description: Round constants (24 x uint64_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint64_t rho8[4] + description: Rotation constant rho8 (4 x uint64_t) + rcx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint64_t rho56[4] + description: Rotation constant rho56 (4 x uint64_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/fips202/x86_64/src/keccak_f1600_x4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(keccak_f1600_x4_avx2_asm) +MLK_ASM_FN_SYMBOL(keccak_f1600_x4_avx2_asm) + + .cfi_startproc + movq %rsp, %r11 + .cfi_def_cfa_register %r11 + andq $-0x20, %rsp + subq $0x300, %rsp # imm = 0x300 + vmovdqu (%rdi), %ymm0 + vmovdqu 0xc8(%rdi), %ymm3 + vmovdqu 0x190(%rdi), %ymm1 + vmovdqu 0x258(%rdi), %ymm4 + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x278(%rdi), %ymm4 + vmovdqu %ymm3, 0x40(%rsp) + vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, (%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0x20(%rdi), %ymm0 + vmovdqu 0x1b0(%rdi), %ymm1 + vmovdqu %ymm3, 0x60(%rsp) + vmovdqu 0xe8(%rdi), %ymm3 + vmovdqu %ymm7, 0x20(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x298(%rdi), %ymm4 + vperm2i128 $0x31, %ymm1, %ymm0, %ymm14 # ymm14 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, 0x80(%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0x40(%rdi), %ymm0 + vmovdqu 0x1d0(%rdi), %ymm1 + vmovdqu %ymm3, 0xc0(%rsp) + vmovdqu 0x108(%rdi), %ymm3 + vmovdqu %ymm14, %ymm10 + vmovdqu %ymm7, 0xa0(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm11 # ymm11 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu %ymm3, 0x100(%rsp) + vperm2i128 $0x31, %ymm1, %ymm0, %ymm8 # ymm8 = ymm0[2,3],ymm1[2,3] + vmovdqu 0x128(%rdi), %ymm3 + vmovdqu 0x60(%rdi), %ymm0 + vmovdqu 0x1f0(%rdi), %ymm1 + vmovdqu %ymm7, 0xe0(%rsp) + vmovdqu %ymm11, %ymm14 + vmovdqu 0x2b8(%rdi), %ymm4 + vmovdqu 0x2f8(%rdi), %ymm5 + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vmovdqu 0x2d8(%rdi), %ymm4 + vperm2i128 $0x20, %ymm3, %ymm2, %ymm15 # ymm15 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm3 # ymm3 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm9 # ymm9 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm3, 0x140(%rsp) + vmovdqu 0x80(%rdi), %ymm0 + vmovdqu 0x148(%rdi), %ymm3 + vmovdqu 0x210(%rdi), %ymm1 + vmovdqu %ymm7, 0x120(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm1, %ymm3 # ymm3 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3] + vperm2i128 $0x20, %ymm3, %ymm2, %ymm7 # ymm7 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm13 # ymm13 = ymm2[2,3],ymm3[2,3] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm7, 0x160(%rsp) + vperm2i128 $0x20, %ymm1, %ymm0, %ymm7 # ymm7 = ymm0[0,1],ymm1[0,1] + vmovdqu 0xa0(%rdi), %ymm0 + vmovdqu 0x230(%rdi), %ymm1 + vmovdqu %ymm3, 0x1a0(%rsp) + vmovdqu 0x168(%rdi), %ymm3 + vpunpcklqdq %ymm5, %ymm1, %ymm4 # ymm4 = ymm1[0],ymm5[0],ymm1[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm1, %ymm1 # ymm1 = ymm1[1],ymm5[1],ymm1[3],ymm5[3] + vmovdqu %ymm7, 0x180(%rsp) + vpunpcklqdq %ymm3, %ymm0, %ymm2 # ymm2 = ymm0[0],ymm3[0],ymm0[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm12 # ymm12 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm1, %ymm0, %ymm3 # ymm3 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm7 # ymm7 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[2,3],ymm1[2,3] + vmovq 0x250(%rdi), %xmm0 + vmovq 0xc0(%rdi), %xmm1 + vmovdqu %ymm12, 0x1c0(%rsp) + vmovdqu %ymm4, 0x1e0(%rsp) + vpinsrq $0x1, 0x318(%rdi), %xmm0, %xmm0 + vpinsrq $0x1, 0x188(%rdi), %xmm1, %xmm1 + vinserti128 $0x1, %xmm0, %ymm1, %ymm2 + movq $0x0, %r10 + +LLmlk_keccak_f1600_x4_avx2: + vmovdqu 0xa0(%rsp), %ymm4 + vpxor 0x1c0(%rsp), %ymm9, %ymm0 + vmovdqu %ymm9, 0x200(%rsp) + vmovdqu %ymm10, %ymm9 + vmovdqu 0xc0(%rsp), %ymm11 + vmovdqu 0x160(%rsp), %ymm12 + vmovdqu %ymm3, 0x240(%rsp) + vpxor 0x100(%rsp), %ymm4, %ymm1 + vmovdqu 0x40(%rsp), %ymm10 + vmovdqu %ymm4, 0x220(%rsp) + vpxor %ymm3, %ymm12, %ymm12 + vmovdqu 0x20(%rsp), %ymm6 + vmovdqu 0x140(%rsp), %ymm4 + vmovdqu %ymm14, 0x2a0(%rsp) + vpxor %ymm1, %ymm0, %ymm0 + vpxor %ymm8, %ymm11, %ymm1 + vpxor 0x180(%rsp), %ymm7, %ymm11 + vmovdqu %ymm10, 0x280(%rsp) + vpxor %ymm1, %ymm12, %ymm12 + vpxor %ymm15, %ymm9, %ymm1 + vmovdqu 0xe0(%rsp), %ymm3 + vmovdqu %ymm8, 0x260(%rsp) + vpxor %ymm1, %ymm11, %ymm11 + vpxor 0x120(%rsp), %ymm14, %ymm1 + vpxor %ymm6, %ymm12, %ymm12 + vmovdqu 0x60(%rsp), %ymm8 + vpxor %ymm10, %ymm11, %ymm11 + vpxor 0x1e0(%rsp), %ymm13, %ymm10 + vpxor %ymm4, %ymm3, %ymm3 + vmovdqu %ymm4, 0x2c0(%rsp) + vpsrlq $0x3f, %ymm12, %ymm4 + vpsrlq $0x3f, %ymm11, %ymm5 + vpxor (%rsp), %ymm0, %ymm0 + vpxor %ymm1, %ymm10, %ymm10 + vmovdqu 0x80(%rsp), %ymm1 + vpxor %ymm8, %ymm10, %ymm10 + vmovdqu %ymm1, %ymm14 + vpxor 0x1a0(%rsp), %ymm2, %ymm1 + vmovdqu %ymm14, 0x2e0(%rsp) + vpxor %ymm3, %ymm1, %ymm1 + vpsllq $0x1, %ymm12, %ymm3 + vpor %ymm4, %ymm3, %ymm3 + vpsllq $0x1, %ymm11, %ymm4 + vpxor %ymm14, %ymm1, %ymm1 + vpor %ymm5, %ymm4, %ymm4 + vpsrlq $0x3f, %ymm10, %ymm14 + vpxor %ymm1, %ymm3, %ymm3 + vpsllq $0x1, %ymm10, %ymm5 + vpxor %ymm0, %ymm4, %ymm4 + vpor %ymm14, %ymm5, %ymm5 + vpxor %ymm6, %ymm4, %ymm6 + vpxor %ymm12, %ymm5, %ymm5 + vpsrlq $0x3f, %ymm1, %ymm12 + vpsllq $0x1, %ymm1, %ymm1 + vpxor %ymm7, %ymm5, %ymm7 + vpxor %ymm9, %ymm5, %ymm9 + vpor %ymm12, %ymm1, %ymm1 + vpxor (%rsp), %ymm3, %ymm12 + vpxor %ymm11, %ymm1, %ymm1 + vpsrlq $0x3f, %ymm0, %ymm11 + vpsllq $0x1, %ymm0, %ymm0 + vpxor %ymm13, %ymm1, %ymm13 + vpxor %ymm8, %ymm1, %ymm8 + vpor %ymm11, %ymm0, %ymm0 + vpxor %ymm10, %ymm0, %ymm0 + vpxor 0xc0(%rsp), %ymm4, %ymm10 + vpxor %ymm2, %ymm0, %ymm2 + vpsrlq $0x14, %ymm10, %ymm11 + vpsllq $0x2c, %ymm10, %ymm10 + vpor %ymm11, %ymm10, %ymm10 + vpxor %ymm15, %ymm5, %ymm11 + vpbroadcastq (%rsi), %ymm15 + vpsrlq $0x15, %ymm11, %ymm14 + vpsllq $0x2b, %ymm11, %ymm11 + vpor %ymm14, %ymm11, %ymm11 + vpandn %ymm11, %ymm10, %ymm14 + vpxor %ymm15, %ymm14, %ymm14 + vpxor %ymm12, %ymm14, %ymm15 + vpsrlq $0x2b, %ymm13, %ymm14 + vpsllq $0x15, %ymm13, %ymm13 + vmovdqu %ymm15, (%rsp) + vpor %ymm14, %ymm13, %ymm13 + vpandn %ymm13, %ymm11, %ymm14 + vpxor %ymm10, %ymm14, %ymm15 + vpsrlq $0x32, %ymm2, %ymm14 + vpsllq $0xe, %ymm2, %ymm2 + vmovdqu %ymm15, 0x20(%rsp) + vpor %ymm14, %ymm2, %ymm2 + vpandn %ymm2, %ymm13, %ymm14 + vpxor %ymm11, %ymm14, %ymm11 + vmovdqu %ymm11, 0x40(%rsp) + vpandn %ymm12, %ymm2, %ymm11 + vpandn %ymm10, %ymm12, %ymm12 + vpxor %ymm13, %ymm11, %ymm11 + vmovdqu %ymm11, 0x60(%rsp) + vpxor %ymm2, %ymm12, %ymm11 + vpsrlq $0x24, %ymm8, %ymm2 + vpsllq $0x1c, %ymm8, %ymm8 + vmovdqu %ymm11, 0x80(%rsp) + vpor %ymm2, %ymm8, %ymm8 + vpxor 0xe0(%rsp), %ymm0, %ymm2 + vpsrlq $0x2c, %ymm2, %ymm10 + vpsllq $0x14, %ymm2, %ymm2 + vpor %ymm10, %ymm2, %ymm2 + vpxor 0x100(%rsp), %ymm3, %ymm10 + vpsrlq $0x3d, %ymm10, %ymm11 + vpsllq $0x3, %ymm10, %ymm10 + vpor %ymm11, %ymm10, %ymm10 + vpandn %ymm10, %ymm2, %ymm11 + vpxor %ymm8, %ymm11, %ymm11 + vmovdqu %ymm11, 0xa0(%rsp) + vpxor 0x160(%rsp), %ymm4, %ymm11 + vpsrlq $0x13, %ymm11, %ymm12 + vpsllq $0x2d, %ymm11, %ymm11 + vpor %ymm12, %ymm11, %ymm11 + vpandn %ymm11, %ymm10, %ymm12 + vpxor %ymm2, %ymm12, %ymm12 + vmovdqu %ymm12, 0xc0(%rsp) + vpsrlq $0x3, %ymm7, %ymm12 + vpsllq $0x3d, %ymm7, %ymm7 + vpor %ymm12, %ymm7, %ymm7 + vpandn %ymm7, %ymm11, %ymm12 + vpxor %ymm10, %ymm12, %ymm10 + vpandn %ymm8, %ymm7, %ymm12 + vpandn %ymm2, %ymm8, %ymm8 + vpsrlq $0x3f, %ymm6, %ymm2 + vpsllq $0x1, %ymm6, %ymm6 + vpxor %ymm11, %ymm12, %ymm14 + vpor %ymm2, %ymm6, %ymm6 + vpsrlq $0x3a, %ymm9, %ymm2 + vpxor %ymm7, %ymm8, %ymm12 + vpsllq $0x6, %ymm9, %ymm9 + vmovdqu %ymm12, 0xe0(%rsp) + vpxor 0x1a0(%rsp), %ymm0, %ymm7 + vpor %ymm2, %ymm9, %ymm9 + vpxor 0x120(%rsp), %ymm1, %ymm2 + vpshufb (%rdx), %ymm7, %ymm7 + vpsrlq $0x27, %ymm2, %ymm11 + vpsllq $0x19, %ymm2, %ymm2 + vpor %ymm2, %ymm11, %ymm11 + vpandn %ymm11, %ymm9, %ymm2 + vpandn %ymm7, %ymm11, %ymm8 + vpxor %ymm6, %ymm2, %ymm12 + vpxor 0x1c0(%rsp), %ymm3, %ymm2 + vpxor %ymm9, %ymm8, %ymm8 + vmovdqu %ymm12, 0x100(%rsp) + vpsrlq $0x2e, %ymm2, %ymm12 + vpsllq $0x12, %ymm2, %ymm2 + vpor %ymm2, %ymm12, %ymm2 + vpandn %ymm2, %ymm7, %ymm12 + vpxor %ymm11, %ymm12, %ymm15 + vpandn %ymm6, %ymm2, %ymm11 + vpandn %ymm9, %ymm6, %ymm6 + vpxor %ymm7, %ymm11, %ymm12 + vmovdqu %ymm12, 0x120(%rsp) + vpxor %ymm2, %ymm6, %ymm12 + vpxor 0x2e0(%rsp), %ymm0, %ymm6 + vpxor 0x2c0(%rsp), %ymm0, %ymm0 + vmovdqu %ymm12, 0x140(%rsp) + vpsrlq $0x25, %ymm6, %ymm2 + vpsllq $0x1b, %ymm6, %ymm6 + vpor %ymm6, %ymm2, %ymm2 + vpxor 0x220(%rsp), %ymm3, %ymm6 + vpxor 0x200(%rsp), %ymm3, %ymm3 + vpsrlq $0x1c, %ymm6, %ymm7 + vpsllq $0x24, %ymm6, %ymm6 + vpor %ymm6, %ymm7, %ymm7 + vpxor 0x260(%rsp), %ymm4, %ymm6 + vpxor 0x240(%rsp), %ymm4, %ymm4 + vpsrlq $0x36, %ymm6, %ymm12 + vpsllq $0xa, %ymm6, %ymm6 + vpor %ymm6, %ymm12, %ymm12 + vpxor 0x180(%rsp), %ymm5, %ymm6 + vpxor 0x280(%rsp), %ymm5, %ymm5 + vpandn %ymm12, %ymm7, %ymm9 + vpsrlq $0x31, %ymm6, %ymm11 + vpsllq $0xf, %ymm6, %ymm6 + vpxor %ymm2, %ymm9, %ymm9 + vpor %ymm6, %ymm11, %ymm11 + vpandn %ymm11, %ymm12, %ymm6 + vpxor %ymm7, %ymm6, %ymm6 + vmovdqu %ymm6, 0x160(%rsp) + vpxor 0x1e0(%rsp), %ymm1, %ymm6 + vpxor 0x2a0(%rsp), %ymm1, %ymm1 + vpshufb (%rcx), %ymm6, %ymm6 + vpandn %ymm6, %ymm11, %ymm13 + vpxor %ymm12, %ymm13, %ymm13 + vmovdqu %ymm13, 0x180(%rsp) + vpandn %ymm2, %ymm6, %ymm13 + vpandn %ymm7, %ymm2, %ymm2 + vpxor %ymm6, %ymm2, %ymm2 + vpsrlq $0x3e, %ymm4, %ymm6 + vpxor %ymm11, %ymm13, %ymm13 + vmovdqu %ymm2, 0x1a0(%rsp) + vpsrlq $0x2, %ymm5, %ymm2 + vpsllq $0x3e, %ymm5, %ymm5 + vpor %ymm5, %ymm2, %ymm2 + vpsrlq $0x9, %ymm1, %ymm5 + vpsllq $0x37, %ymm1, %ymm1 + vpsllq $0x2, %ymm4, %ymm4 + vpor %ymm1, %ymm5, %ymm1 + vpsrlq $0x19, %ymm0, %ymm5 + vpor %ymm4, %ymm6, %ymm4 + vpsllq $0x27, %ymm0, %ymm0 + vpor %ymm0, %ymm5, %ymm5 + vpandn %ymm5, %ymm1, %ymm0 + vpxor %ymm2, %ymm0, %ymm0 + vmovdqu %ymm0, 0x1c0(%rsp) + vpsrlq $0x17, %ymm3, %ymm0 + vpsllq $0x29, %ymm3, %ymm3 + vpor %ymm3, %ymm0, %ymm0 + vpandn %ymm4, %ymm0, %ymm7 + vpandn %ymm0, %ymm5, %ymm3 + vpxor %ymm5, %ymm7, %ymm7 + vpandn %ymm2, %ymm4, %ymm5 + vpandn %ymm1, %ymm2, %ymm2 + vpxor %ymm0, %ymm5, %ymm5 + vpxor %ymm1, %ymm3, %ymm3 + vpxor %ymm4, %ymm2, %ymm2 + vmovdqu %ymm5, 0x1e0(%rsp) + addq $0x8, %rsi + addq $0x1, %r10 + cmpq $0x18, %r10 + jne LLmlk_keccak_f1600_x4_avx2 + vmovdqu (%rsp), %ymm4 + vmovdqu 0x40(%rsp), %ymm5 + vmovdqu 0x20(%rsp), %ymm0 + vmovdqu 0x60(%rsp), %ymm1 + vmovdqu 0x1c0(%rsp), %ymm12 + vmovdqu %ymm2, 0x1c0(%rsp) + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vpunpcklqdq %ymm1, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm1[0],ymm5[2],ymm1[2] + vpunpckhqdq %ymm1, %ymm5, %ymm1 # ymm1 = ymm5[1],ymm1[1],ymm5[3],ymm1[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vmovdqu 0x80(%rsp), %ymm4 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm6, (%rdi) + vmovdqu %ymm5, 0xc8(%rdi) + vmovdqu %ymm2, 0x190(%rdi) + vmovdqu %ymm0, 0x258(%rdi) + vmovdqu 0xa0(%rsp), %ymm0 + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm1 # ymm1 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vmovdqu 0xc0(%rsp), %ymm0 + vpunpcklqdq %ymm10, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm10[0],ymm0[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm10[1],ymm0[3],ymm10[3] + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vmovdqu 0xe0(%rsp), %ymm4 + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x100(%rsp), %ymm0 + vmovdqu %ymm2, 0x1b0(%rdi) + vmovdqu %ymm1, 0x278(%rdi) + vpunpcklqdq %ymm4, %ymm14, %ymm2 # ymm2 = ymm14[0],ymm4[0],ymm14[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm14, %ymm1 # ymm1 = ymm14[1],ymm4[1],ymm14[3],ymm4[3] + vpunpcklqdq %ymm8, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm8[0],ymm0[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm8[1],ymm0[3],ymm8[3] + vmovdqu %ymm6, 0x20(%rdi) + vmovdqu %ymm5, 0xe8(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x120(%rsp), %ymm4 + vmovdqu 0x140(%rsp), %ymm0 + vmovdqu %ymm2, 0x1d0(%rdi) + vmovdqu %ymm1, 0x298(%rdi) + vpunpcklqdq %ymm4, %ymm15, %ymm2 # ymm2 = ymm15[0],ymm4[0],ymm15[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm15, %ymm1 # ymm1 = ymm15[1],ymm4[1],ymm15[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm0, %ymm4 # ymm4 = ymm0[0],ymm9[0],ymm0[2],ymm9[2] + vmovdqu %ymm5, 0x108(%rdi) + vpunpckhqdq %ymm9, %ymm0, %ymm0 # ymm0 = ymm0[1],ymm9[1],ymm0[3],ymm9[3] + vmovdqu %ymm6, 0x40(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm0, %ymm1, %ymm5 # ymm5 = ymm1[0,1],ymm0[0,1] + vmovdqu 0x160(%rsp), %ymm4 + vperm2i128 $0x31, %ymm0, %ymm1, %ymm1 # ymm1 = ymm1[2,3],ymm0[2,3] + vmovdqu 0x180(%rsp), %ymm0 + vmovdqu %ymm5, 0x128(%rdi) + vmovdqu 0x1a0(%rsp), %ymm5 + vmovdqu %ymm2, 0x1f0(%rdi) + vpunpcklqdq %ymm0, %ymm4, %ymm2 # ymm2 = ymm4[0],ymm0[0],ymm4[2],ymm0[2] + vpunpckhqdq %ymm0, %ymm4, %ymm0 # ymm0 = ymm4[1],ymm0[1],ymm4[3],ymm0[3] + vpunpcklqdq %ymm5, %ymm13, %ymm4 # ymm4 = ymm13[0],ymm5[0],ymm13[2],ymm5[2] + vmovdqu %ymm6, 0x60(%rdi) + vperm2i128 $0x20, %ymm4, %ymm2, %ymm6 # ymm6 = ymm2[0,1],ymm4[0,1] + vmovdqu %ymm1, 0x2b8(%rdi) + vperm2i128 $0x31, %ymm4, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm4[2,3] + vpunpckhqdq %ymm5, %ymm13, %ymm1 # ymm1 = ymm13[1],ymm5[1],ymm13[3],ymm5[3] + vmovdqu %ymm6, 0x80(%rdi) + vmovdqu 0x1e0(%rsp), %ymm4 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm5 # ymm5 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm2, 0x210(%rdi) + vpunpcklqdq %ymm3, %ymm12, %ymm2 # ymm2 = ymm12[0],ymm3[0],ymm12[2],ymm3[2] + vmovdqu %ymm0, 0x2d8(%rdi) + vpunpckhqdq %ymm3, %ymm12, %ymm0 # ymm0 = ymm12[1],ymm3[1],ymm12[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm4[0],ymm7[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm7, %ymm1 # ymm1 = ymm7[1],ymm4[1],ymm7[3],ymm4[3] + vmovdqu %ymm5, 0x148(%rdi) + vperm2i128 $0x20, %ymm3, %ymm2, %ymm5 # ymm5 = ymm2[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm2, %ymm2 # ymm2 = ymm2[2,3],ymm3[2,3] + vmovdqu 0x1c0(%rsp), %ymm3 + vperm2i128 $0x20, %ymm1, %ymm0, %ymm4 # ymm4 = ymm0[0,1],ymm1[0,1] + vperm2i128 $0x31, %ymm1, %ymm0, %ymm0 # ymm0 = ymm0[2,3],ymm1[2,3] + vmovdqu %ymm5, 0xa0(%rdi) + vextracti128 $0x1, %ymm3, %xmm15 + vmovdqu %ymm4, 0x168(%rdi) + vmovdqu %ymm2, 0x230(%rdi) + vmovdqu %ymm0, 0x2f8(%rdi) + vmovq %xmm3, 0xc0(%rdi) + vmovhpd %xmm3, 0x188(%rdi) + vmovq %xmm15, 0x250(%rdi) + vmovhpd %xmm15, 0x318(%rdi) + movq %r11, %rsp + .cfi_def_cfa_register %rsp + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(keccak_f1600_x4_avx2_asm) + +#endif /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c new file mode 100644 index 0000000000..1fb58915f0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/fips202/native/x86_64/src/keccakf1600_constants.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../../common.h" +#if defined(MLK_FIPS202_X86_64_NEED_X4_AVX2) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include + +#include "fips202_native_x86_64.h" + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t + mlk_keccakf1600_round_constants[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho8[4] = { + 0x0605040302010007, + 0x0e0d0c0b0a09080f, + 0x1615141312111017, + 0x1e1d1c1b1a19181f, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint64_t mlk_keccak_rho56[4] = { + 0x0007060504030201, + 0x080f0e0d0c0b0a09, + 0x1017161514131211, + 0x181f1e1d1c1b1a19, +}; + +#else /* MLK_FIPS202_X86_64_NEED_X4_AVX2 && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(fips202_x86_64_constants) + +#endif /* !(MLK_FIPS202_X86_64_NEED_X4_AVX2 && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/indcpa.c b/src/mlkem-native/mlkem/src/indcpa.c new file mode 100644 index 0000000000..98060e3742 --- /dev/null +++ b/src/mlkem-native/mlkem/src/indcpa.c @@ -0,0 +1,675 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "indcpa.h" + +#include "debug.h" +#include "randombytes.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_pack_pk MLK_ADD_PARAM_SET(mlk_pack_pk) +#define mlk_unpack_pk MLK_ADD_PARAM_SET(mlk_unpack_pk) +#define mlk_pack_sk MLK_ADD_PARAM_SET(mlk_pack_sk) +#define mlk_unpack_sk MLK_ADD_PARAM_SET(mlk_unpack_sk) +#define mlk_pack_ciphertext MLK_ADD_PARAM_SET(mlk_pack_ciphertext) +#define mlk_unpack_ciphertext MLK_ADD_PARAM_SET(mlk_unpack_ciphertext) +#define mlk_matvec_mul MLK_ADD_PARAM_SET(mlk_matvec_mul) +#define mlk_polyvec_permute_bitrev_to_custom \ + MLK_ADD_PARAM_SET(mlk_polyvec_permute_bitrev_to_custom) +#define mlk_polymat_permute_bitrev_to_custom \ + MLK_ADD_PARAM_SET(mlk_polymat_permute_bitrev_to_custom) +#define mlk_keypair_getnoise_eta1 MLK_ADD_PARAM_SET(mlk_keypair_getnoise_eta1) +#define mlk_enc_getnoise_eta1_eta2 MLK_ADD_PARAM_SET(mlk_enc_getnoise_eta1_eta2) +/* End of parameter set namespacing */ + +/** + * Serialize the public key as the concatenation of the serialized vector of + * polynomials pk and the public seed used to generate the matrix A. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L19].} + * + * @param[out] r Output serialized public key. + * @param[in] pk Input public-key polyvec. Must have coefficients within + * [0,..,MLKEM_Q-1]. + * @param[in] seed Input public seed. + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +static void mlk_pack_pk(uint8_t r[MLKEM_INDCPA_PUBLICKEYBYTES], + const mlk_polyvec *pk, + const uint8_t seed[MLKEM_SYMBYTES]) +{ + mlk_assert_bound_2d(pk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + mlk_polyvec_tobytes(r, pk); + mlk_memcpy(r + MLKEM_POLYVECBYTES, seed, MLKEM_SYMBYTES); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * De-serialize public key from a byte array; approximate inverse of + * mlk_pack_pk. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L2-3].} + * + * @param[out] pk Output public-key polynomial vector. Coefficients + * will be normalized to [0,1,..,MLKEM_Q-1]. + * @param[out] seed Output seed to generate matrix A. + * @param[in] packedpk Input serialized public key. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_unpack_pk(mlk_polyvec *pk, uint8_t seed[MLKEM_SYMBYTES], + const uint8_t packedpk[MLKEM_INDCPA_PUBLICKEYBYTES]) +{ + mlk_polyvec_frombytes(pk, packedpk); + mlk_memcpy(seed, packedpk + MLKEM_POLYVECBYTES, MLKEM_SYMBYTES); + + /* NOTE: If a modulus check was conducted on the PK, we know at this + * point that the coefficients of `pk` are unsigned canonical. The + * specifications and proofs, however, do _not_ assume this, and instead + * work with the easily provable bound by MLKEM_UINT12_LIMIT. */ +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/** + * Serialize the secret key. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L20].} + * + * @param[out] r Output serialized secret key. + * @param[in] sk Input vector of polynomials (secret key). + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +static void mlk_pack_sk(uint8_t r[MLKEM_INDCPA_SECRETKEYBYTES], + const mlk_polyvec *sk) +{ + mlk_assert_bound_2d(sk->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + mlk_polyvec_tobytes(r, sk); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * De-serialize the secret key; inverse of mlk_pack_sk. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L5].} + * + * @param[out] sk Output vector of polynomials (secret key). + * @param[in] packedsk Input serialized secret key. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_unpack_sk(mlk_polyvec *sk, + const uint8_t packedsk[MLKEM_INDCPA_SECRETKEYBYTES]) +{ + mlk_polyvec_frombytes(sk, packedsk); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/** + * Serialize the ciphertext as the concatenation of the compressed and + * serialized vector of polynomials b and the compressed and serialized + * polynomial v. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L22-23].} + * + * @param[out] r Output serialized ciphertext. + * @param[in] b Input vector of polynomials b. + * @param[in] v Input polynomial v. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_pack_ciphertext(uint8_t r[MLKEM_INDCPA_BYTES], + const mlk_polyvec *b, mlk_poly *v) +{ + mlk_polyvec_compress_du(r, b); + mlk_poly_compress_dv(r + MLKEM_POLYVECCOMPRESSEDBYTES_DU, v); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/** + * De-serialize and decompress ciphertext from a byte array; approximate + * inverse of mlk_pack_ciphertext. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt), L1-4].} + * + * @param[out] b Output vector of polynomials b. + * @param[out] v Output polynomial v. + * @param[in] c Input serialized ciphertext. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_unpack_ciphertext(mlk_polyvec *b, mlk_poly *v, + const uint8_t c[MLKEM_INDCPA_BYTES]) +{ + mlk_polyvec_decompress_du(b, c); + mlk_poly_decompress_dv(v, c + MLKEM_POLYVECCOMPRESSEDBYTES_DU); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* Helper function to ensure that the polynomial entries in the output + * of gen_matrix use the standard (bitreversed) ordering of coefficients. + * No-op unless a native backend with a custom ordering is used. + * + * We don't inline this into gen_matrix to avoid having to split the CBMC + * proof for gen_matrix based on MLK_USE_NATIVE_NTT_CUSTOM_ORDER. */ +static void mlk_polyvec_permute_bitrev_to_custom(mlk_polyvec *v) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(v, sizeof(mlk_polyvec))) + requires(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(v, sizeof(mlk_polyvec))) + ensures(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) +{ +#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER) + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(v, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + invariant(forall(x, 0, MLKEM_K, + array_bound(v->vec[x].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + decreases(MLKEM_K - i)) + { + mlk_poly_permute_bitrev_to_custom(v->vec[i].coeffs); + } +#else /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ + /* Nothing to do */ + (void)v; +#endif /* !MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ +} + +static void mlk_polymat_permute_bitrev_to_custom(mlk_polymat *a) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) + assigns(memory_slice(a, sizeof(mlk_polymat))) + ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q))))) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(a, sizeof(mlk_polymat))) + invariant(i <= MLKEM_K) + invariant(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) + decreases(MLKEM_K - i)) + { + mlk_polyvec_permute_bitrev_to_custom(&a->vec[i]); + } +} + +/* Reference: `gen_matrix()` in the reference implementation @[REF]. + * - We use a special subroutine to generate 4 polynomials + * at a time, to be able to leverage batched Keccak-f1600 + * implementations. The reference implementation generates + * one matrix entry a time. + * + * Not static for benchmarking */ +MLK_INTERNAL_API +void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES], + int transposed) +{ + unsigned i, j; + MLK_ALIGN uint8_t seed_ext[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]; + + for (j = 0; j < 4; j++) + { + mlk_memcpy(seed_ext[j], seed, MLKEM_SYMBYTES); + } + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + /* Sample 4 matrix entries a time. */ + for (i = 0; i < (MLKEM_K * MLKEM_K / 4) * 4; i += 4) + { + for (j = 0; j < 4; j++) + { + uint8_t x, y; + /* MLKEM_K <= 4, so the values fit in uint8_t. */ + x = (uint8_t)((i + j) / MLKEM_K); + y = (uint8_t)((i + j) % MLKEM_K); + if (transposed) + { + seed_ext[j][MLKEM_SYMBYTES + 0] = x; + seed_ext[j][MLKEM_SYMBYTES + 1] = y; + } + else + { + seed_ext[j][MLKEM_SYMBYTES + 0] = y; + seed_ext[j][MLKEM_SYMBYTES + 1] = x; + } + } + + mlk_poly_rej_uniform_x4(&a->vec[i / MLKEM_K].vec[i % MLKEM_K], + &a->vec[(i + 1) / MLKEM_K].vec[(i + 1) % MLKEM_K], + &a->vec[(i + 2) / MLKEM_K].vec[(i + 2) % MLKEM_K], + &a->vec[(i + 3) / MLKEM_K].vec[(i + 3) % MLKEM_K], + seed_ext); + } +#else /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + /* When using serial FIPS202, sample all entries individually. */ + i = 0; +#endif /* MLK_CONFIG_SERIAL_FIPS202_ONLY */ + + /* For MLKEM_K == 3, sample the last entry individually. + * When MLK_CONFIG_SERIAL_FIPS202_ONLY is set, sample all entries + * individually. */ + for (; i < MLKEM_K * MLKEM_K; i++) + { + uint8_t x, y; + /* MLKEM_K <= 4, so the values fit in uint8_t. */ + x = (uint8_t)(i / MLKEM_K); + y = (uint8_t)(i % MLKEM_K); + + if (transposed) + { + seed_ext[0][MLKEM_SYMBYTES + 0] = x; + seed_ext[0][MLKEM_SYMBYTES + 1] = y; + } + else + { + seed_ext[0][MLKEM_SYMBYTES + 0] = y; + seed_ext[0][MLKEM_SYMBYTES + 1] = x; + } + + mlk_poly_rej_uniform(&a->vec[i / MLKEM_K].vec[i % MLKEM_K], seed_ext[0]); + } + + mlk_assert(i == MLKEM_K * MLKEM_K); + + /* + * The public matrix is generated in NTT domain. If the native backend + * uses a custom order in NTT domain, permute A accordingly. + */ + mlk_polymat_permute_bitrev_to_custom(a); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(seed_ext, sizeof(seed_ext)); +} + +/** + * Compute matrix-vector product in NTT domain, via Montgomery multiplication. + * + * @spec{Implements @[FIPS203, Section 2.4.7, Eq (2.12), (2.13)].} + * + * @param[out] out Output polynomial vector. + * @param[in] a Input matrix. Must be in NTT domain and have coefficients + * of absolute value < 4096. + * @param[in] v Input polynomial vector. Must be in NTT domain. + * @param[in] vc Mulcache for @p v, computed via + * mlk_polyvec_mulcache_compute(). + */ +static void mlk_matvec_mul(mlk_polyvec *out, const mlk_polymat *a, + const mlk_polyvec *v, const mlk_polyvec_mulcache *vc) +__contract__( + requires(memory_no_alias(out, sizeof(mlk_polyvec))) + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(memory_no_alias(v, sizeof(mlk_polyvec))) + requires(memory_no_alias(vc, sizeof(mlk_polyvec_mulcache))) + requires(forall(k0, 0, MLKEM_K, + forall(k1, 0, MLKEM_K, + array_bound(a->vec[k0].vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)))) + assigns(memory_slice(out, sizeof(mlk_polyvec)))) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(out, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + decreases(MLKEM_K - i)) + { + mlk_polyvec_basemul_acc_montgomery_cached(&out->vec[i], &a->vec[i], v, vc); + } +} + +/** + * Compute and fill the pv and e polyvec structures needed by + * mlk_keypair_derand(). Uses x4-batched versions of `poly_getnoise` to + * leverage batched Keccak-f1600. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)] steps 8-15.} + * + * @param[out] pv Output polynomial vector. + * @param[out] e Output polynomial vector. + * @param[in] seed Seed bytes for sampling. + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +static void mlk_keypair_getnoise_eta1(mlk_polyvec *pv, mlk_polyvec *e, + const uint8_t seed[MLKEM_SYMBYTES]) +__contract__( + requires(memory_no_alias(pv, sizeof(mlk_polyvec))) + requires(memory_no_alias(e, sizeof(mlk_polyvec))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(pv, sizeof(mlk_polyvec))) + assigns(memory_slice(e, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, array_abs_bound(pv->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) + ensures(forall(k1, 0, MLKEM_K, array_abs_bound(e->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) +) +{ +#if MLKEM_K == 2 + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], /* Fill elements of pv */ + &e->vec[0], &e->vec[1], /* and two elements of e */ + seed, 0, 1, 2, 3); +#elif MLKEM_K == 3 + /* + * Only the first three output buffers are needed, so we pass NULL as + * the fourth parameter, and 0xFF as its dummy nonce. + */ + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], NULL, seed, + 0, 1, 2, 0xFF); + /* Same here */ + mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], NULL, seed, 3, + 4, 5, 0xFF); +#elif MLKEM_K == 4 + mlk_poly_getnoise_eta1_4x(&pv->vec[0], &pv->vec[1], &pv->vec[2], &pv->vec[3], + seed, 0, 1, 2, 3); + mlk_poly_getnoise_eta1_4x(&e->vec[0], &e->vec[1], &e->vec[2], &e->vec[3], + seed, 4, 5, 6, 7); +#endif /* MLKEM_K == 4 */ +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Compute and fill the sp, ep, and epp polynomial structures needed by + * mlk_indcpa_enc(). Uses x4-batched versions of `poly_getnoise` to leverage + * batched Keccak-f1600. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)] steps 9-16.} + * + * @param[out] sp Output polynomial vector. + * @param[out] ep Output polynomial vector. + * @param[out] epp Output polynomial. + * @param[in] coins Seed bytes for sampling. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +static void mlk_enc_getnoise_eta1_eta2(mlk_polyvec *sp, mlk_polyvec *ep, + mlk_poly *epp, + const uint8_t coins[MLKEM_SYMBYTES]) +__contract__( + requires(memory_no_alias(sp, sizeof(mlk_polyvec))) + requires(memory_no_alias(ep, sizeof(mlk_polyvec))) + requires(memory_no_alias(epp, sizeof(mlk_poly))) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(sp, sizeof(mlk_polyvec))) + assigns(memory_slice(ep, sizeof(mlk_polyvec))) + assigns(memory_slice(epp, sizeof(mlk_poly))) + ensures(forall(k0, 0, MLKEM_K, array_abs_bound(sp->vec[k0].coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1))) + ensures(forall(k1, 0, MLKEM_K, array_abs_bound(ep->vec[k1].coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1))) + ensures(array_abs_bound(epp->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1)) +) +{ +#if MLKEM_K == 2 + mlk_poly_getnoise_eta1122_4x(&sp->vec[0], &sp->vec[1], &ep->vec[0], + &ep->vec[1], coins, 0, 1, 2, 3); + mlk_poly_getnoise_eta2(epp, coins, 4); +#elif MLKEM_K == 3 + /* + * In this call, only the first three output buffers are needed, + * so we pass NULL as the fourth parameter, and 0xFF as its dummy nonce. + */ + mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], NULL, coins, + 0, 1, 2, 0xFF /* irrelevant */); + /* The fourth output buffer in this call _is_ used. */ + mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], epp, coins, + 3, 4, 5, 6); +#elif MLKEM_K == 4 + mlk_poly_getnoise_eta1_4x(&sp->vec[0], &sp->vec[1], &sp->vec[2], &sp->vec[3], + coins, 0, 1, 2, 3); + mlk_poly_getnoise_eta2_4x(&ep->vec[0], &ep->vec[1], &ep->vec[2], &ep->vec[3], + coins, 4, 5, 6, 7); + mlk_poly_getnoise_eta2(epp, coins, 8); +#endif /* MLKEM_K == 4 */ +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + + +/* Reference: `indcpa_keypair_derand()` in the reference implementation @[REF]. + * - We use a different implementation of `gen_matrix()` which + * uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above). + * - We use a mulcache to speed up matrix-vector multiplication. + * - We include buffer zeroization. + */ +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_INTERNAL_API +int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + const uint8_t *publicseed; + const uint8_t *noiseseed; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context); + MLK_ALLOC(a, mlk_polymat, 1, context); + MLK_ALLOC(e, mlk_polyvec, 1, context); + MLK_ALLOC(pkpv, mlk_polyvec, 1, context); + MLK_ALLOC(skpv, mlk_polyvec, 1, context); + MLK_ALLOC(skpv_cache, mlk_polyvec_mulcache, 1, context); + + if (buf == NULL || coins_with_domain_separator == NULL || a == NULL || + e == NULL || pkpv == NULL || skpv == NULL || skpv_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + publicseed = buf; + noiseseed = buf + MLKEM_SYMBYTES; + + /* Concatenate coins with MLKEM_K for domain separation of security levels */ + mlk_memcpy(coins_with_domain_separator, coins, MLKEM_SYMBYTES); + coins_with_domain_separator[MLKEM_SYMBYTES] = MLKEM_K; + + mlk_hash_g(buf, coins_with_domain_separator, MLKEM_SYMBYTES + 1); + + /* + * Declassify the public seed. + * Required to use it in conditional-branches in rejection sampling. + * This is needed because all output of randombytes is marked as secret + * (=undefined) + */ + MLK_CT_TESTING_DECLASSIFY(publicseed, MLKEM_SYMBYTES); + + mlk_gen_matrix(a, publicseed, 0 /* no transpose */); + + mlk_keypair_getnoise_eta1(skpv, e, noiseseed); + + mlk_polyvec_ntt(skpv); + mlk_polyvec_ntt(e); + + mlk_polyvec_mulcache_compute(skpv_cache, skpv); + mlk_matvec_mul(pkpv, a, skpv, skpv_cache); + mlk_polyvec_tomont(pkpv); + + mlk_polyvec_add(pkpv, e); + mlk_polyvec_reduce(pkpv); + mlk_polyvec_reduce(skpv); + + mlk_pack_sk(sk, skpv); + mlk_pack_pk(pk, pkpv, publicseed); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(skpv_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(skpv, mlk_polyvec, 1, context); + MLK_FREE(pkpv, mlk_polyvec, 1, context); + MLK_FREE(e, mlk_polyvec, 1, context); + MLK_FREE(a, mlk_polymat, 1, context); + MLK_FREE(coins_with_domain_separator, uint8_t, MLKEM_SYMBYTES + 1, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/* Reference: `indcpa_enc()` in the reference implementation @[REF]. + * - We use x4-batched versions of `poly_getnoise` to leverage + * batched x4-batched Keccak-f1600. + * - We use a different implementation of `gen_matrix()` which + * uses x4-batched Keccak-f1600 (see `mlk_gen_matrix()` above). + * - We use a mulcache to speed up matrix-vector multiplication. + * - We include buffer zeroization. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_INTERNAL_API +int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(seed, uint8_t, MLKEM_SYMBYTES, context); + MLK_ALLOC(at, mlk_polymat, 1, context); + MLK_ALLOC(sp, mlk_polyvec, 1, context); + MLK_ALLOC(pkpv, mlk_polyvec, 1, context); + MLK_ALLOC(ep, mlk_polyvec, 1, context); + MLK_ALLOC(b, mlk_polyvec, 1, context); + MLK_ALLOC(v, mlk_poly, 1, context); + MLK_ALLOC(k, mlk_poly, 1, context); + MLK_ALLOC(epp, mlk_poly, 1, context); + MLK_ALLOC(sp_cache, mlk_polyvec_mulcache, 1, context); + + if (seed == NULL || at == NULL || sp == NULL || pkpv == NULL || ep == NULL || + b == NULL || v == NULL || k == NULL || epp == NULL || sp_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_unpack_pk(pkpv, seed, pk); + mlk_poly_frommsg(k, m); + + /* + * Declassify the public seed. + * Required to use it in conditional-branches in rejection sampling. + * This is needed because in re-encryption the publicseed originated from sk + * which is marked undefined. + */ + MLK_CT_TESTING_DECLASSIFY(seed, MLKEM_SYMBYTES); + + mlk_gen_matrix(at, seed, 1 /* transpose */); + + mlk_enc_getnoise_eta1_eta2(sp, ep, epp, coins); + + mlk_polyvec_ntt(sp); + + mlk_polyvec_mulcache_compute(sp_cache, sp); + mlk_matvec_mul(b, at, sp, sp_cache); + mlk_polyvec_basemul_acc_montgomery_cached(v, pkpv, sp, sp_cache); + + mlk_polyvec_invntt_tomont(b); + mlk_poly_invntt_tomont(v); + + mlk_polyvec_add(b, ep); + mlk_poly_add(v, epp); + mlk_poly_add(v, k); + + mlk_polyvec_reduce(b); + mlk_poly_reduce(v); + + mlk_pack_ciphertext(c, b, v); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(sp_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(epp, mlk_poly, 1, context); + MLK_FREE(k, mlk_poly, 1, context); + MLK_FREE(v, mlk_poly, 1, context); + MLK_FREE(b, mlk_polyvec, 1, context); + MLK_FREE(ep, mlk_polyvec, 1, context); + MLK_FREE(pkpv, mlk_polyvec, 1, context); + MLK_FREE(sp, mlk_polyvec, 1, context); + MLK_FREE(at, mlk_polymat, 1, context); + MLK_FREE(seed, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* Reference: `indcpa_dec()` in the reference implementation @[REF]. + * - We use a mulcache for the scalar product. + * - We include buffer zeroization. */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_INTERNAL_API +int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(b, mlk_polyvec, 1, context); + MLK_ALLOC(skpv, mlk_polyvec, 1, context); + MLK_ALLOC(v, mlk_poly, 1, context); + MLK_ALLOC(sb, mlk_poly, 1, context); + MLK_ALLOC(b_cache, mlk_polyvec_mulcache, 1, context); + + if (b == NULL || skpv == NULL || v == NULL || sb == NULL || b_cache == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_unpack_ciphertext(b, v, c); + mlk_unpack_sk(skpv, sk); + + mlk_polyvec_ntt(b); + mlk_polyvec_mulcache_compute(b_cache, b); + mlk_polyvec_basemul_acc_montgomery_cached(sb, skpv, b, b_cache); + mlk_poly_invntt_tomont(sb); + + mlk_poly_sub(v, sb); + mlk_poly_reduce(v); + + mlk_poly_tomsg(m, v); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(b_cache, mlk_polyvec_mulcache, 1, context); + MLK_FREE(sb, mlk_poly, 1, context); + MLK_FREE(v, mlk_poly, 1, context); + MLK_FREE(skpv, mlk_polyvec, 1, context); + MLK_FREE(b, mlk_polyvec, 1, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_pack_pk +#undef mlk_unpack_pk +#undef mlk_pack_sk +#undef mlk_unpack_sk +#undef mlk_pack_ciphertext +#undef mlk_unpack_ciphertext +#undef mlk_matvec_mul +#undef mlk_polyvec_permute_bitrev_to_custom +#undef mlk_polymat_permute_bitrev_to_custom +#undef mlk_keypair_getnoise_eta1 +#undef mlk_enc_getnoise_eta1_eta2 diff --git a/src/mlkem-native/mlkem/src/indcpa.h b/src/mlkem-native/mlkem/src/indcpa.h new file mode 100644 index 0000000000..82315c2ef2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/indcpa.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_INDCPA_H +#define MLK_INDCPA_H + +#include "cbmc.h" +#include "common.h" +#include "poly_k.h" + +#define mlk_gen_matrix MLK_NAMESPACE_K(gen_matrix) +/** + * Deterministically generate matrix A (or the transpose of A) from a seed. + * Entries of the matrix are polynomials that look uniformly random. + * Performs rejection sampling on the output of an XOF. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen), L3-7] and + * @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L4-8]. The @p transposed + * parameter only affects internal presentation.} + * + * @param[out] a Output matrix A. + * @param[in] seed Input seed. + * @param transposed Boolean deciding whether A or A^T is generated. + */ +MLK_INTERNAL_API +void mlk_gen_matrix(mlk_polymat *a, const uint8_t seed[MLKEM_SYMBYTES], + int transposed) +__contract__( + requires(memory_no_alias(a, sizeof(mlk_polymat))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + requires(transposed == 0 || transposed == 1) + assigns(memory_slice(a, sizeof(mlk_polymat))) + ensures(forall(x, 0, MLKEM_K, forall(y, 0, MLKEM_K, + array_bound(a->vec[x].vec[y].coeffs, 0, MLKEM_N, 0, MLKEM_Q)))) +); + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_indcpa_keypair_derand \ + MLK_NAMESPACE_K(indcpa_keypair_derand) MLK_CONTEXT_PARAMETERS_3 +/** + * Generate public and private key for the CPA-secure public-key encryption + * scheme underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 13 (K-PKE.KeyGen)].} + * + * @param[out] pk Output public key + * (length MLKEM_INDCPA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key + * (length MLKEM_INDCPA_SECRETKEYBYTES bytes). + * @param[in] coins Input randomness (length MLKEM_SYMBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_keypair_derand(uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(sk, MLKEM_INDCPA_SECRETKEYBYTES)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_indcpa_enc MLK_NAMESPACE_K(indcpa_enc) MLK_CONTEXT_PARAMETERS_4 +/** + * Encryption function of the CPA-secure public-key encryption scheme + * underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 14 (K-PKE.Encrypt)].} + * + * @param[out] c Output ciphertext (length MLKEM_INDCPA_BYTES bytes). + * @param[in] m Input message (length MLKEM_INDCPA_MSGBYTES bytes). + * @param[in] pk Input public key + * (length MLKEM_INDCPA_PUBLICKEYBYTES bytes). + * @param[in] coins Input random coins used as seed (length MLKEM_SYMBYTES + * bytes) to deterministically generate all randomness. + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_enc(uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t pk[MLKEM_INDCPA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(c, MLKEM_INDCPA_BYTES)) + requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCPA_PUBLICKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(c, MLKEM_INDCPA_BYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(c, MLKEM_INDCPA_BYTES)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_indcpa_dec MLK_NAMESPACE_K(indcpa_dec) MLK_CONTEXT_PARAMETERS_3 +/** + * Decryption function of the CPA-secure public-key encryption scheme + * underlying ML-KEM. + * + * @spec{Implements @[FIPS203, Algorithm 15 (K-PKE.Decrypt)].} + * + * @param[out] m Output decrypted message + * (length MLKEM_INDCPA_MSGBYTES bytes). + * @param[in] c Input ciphertext (length MLKEM_INDCPA_BYTES bytes). + * @param[in] sk Input secret key + * (length MLKEM_INDCPA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +MLK_INTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_indcpa_dec(uint8_t m[MLKEM_INDCPA_MSGBYTES], + const uint8_t c[MLKEM_INDCPA_BYTES], + const uint8_t sk[MLKEM_INDCPA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(c, MLKEM_INDCPA_BYTES)) + requires(memory_no_alias(m, MLKEM_INDCPA_MSGBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCPA_SECRETKEYBYTES)) + assigns(memory_slice(m, MLKEM_INDCPA_MSGBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(m, MLKEM_INDCPA_MSGBYTES)) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_INDCPA_H */ diff --git a/src/mlkem-native/mlkem/src/kem.c b/src/mlkem-native/mlkem/src/kem.c new file mode 100644 index 0000000000..57e767fd51 --- /dev/null +++ b/src/mlkem-native/mlkem/src/kem.c @@ -0,0 +1,474 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS140_3_IG] + * Implementation Guidance for FIPS 140-3 and the Cryptographic Module + * Validation Program + * National Institute of Standards and Technology + * https://csrc.nist.gov/projects/cryptographic-module-validation-program/fips-140-3-ig-announcements + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "kem.h" + +#include "indcpa.h" +#include "randombytes.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying security levels) + * within a single compilation unit. */ +#define mlk_check_pct MLK_ADD_PARAM_SET(mlk_check_pct) MLK_CONTEXT_PARAMETERS_2 +/* End of parameter set namespacing */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: Not implemented in the reference implementation @[REF]. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(p, mlk_polyvec, 1, context); + MLK_ALLOC(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context); + + if (p == NULL || p_reencoded == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + mlk_polyvec_frombytes(p, pk); + mlk_polyvec_reduce(p); + mlk_polyvec_tobytes(p_reencoded, p); + + /* We use a constant-time memcmp here to avoid having to + * declassify the PK before the PCT has succeeded. */ + ret = mlk_ct_memcmp(pk, p_reencoded, MLKEM_POLYVECBYTES) ? MLK_ERR_INVALID_PK + : 0; + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(p_reencoded, uint8_t, MLKEM_POLYVECBYTES, context); + MLK_FREE(p, mlk_polyvec, 1, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: Not implemented in the reference implementation @[REF]. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(test, uint8_t, MLKEM_SYMBYTES, context); + + if (test == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* + * The parts of `sk` being hashed and compared here are public, so + * no public information is leaked through the runtime or the return value + * of this function. + */ + + /* Declassify the public part of the secret key */ + MLK_CT_TESTING_DECLASSIFY(sk + MLKEM_INDCPA_SECRETKEYBYTES, + MLKEM_INDCCA_PUBLICKEYBYTES); + MLK_CT_TESTING_DECLASSIFY( + sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, MLKEM_SYMBYTES); + + mlk_hash_h(test, sk + MLKEM_INDCPA_SECRETKEYBYTES, + MLKEM_INDCCA_PUBLICKEYBYTES); + /* This doesn't have to be a constant-time memcmp, but it's the only place + * in the library where a normal memcmp would be used otherwise, so for sake + * of minimizing stdlib dependency, we use our constant-time one anyway. */ + ret = mlk_ct_memcmp(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, + test, MLKEM_SYMBYTES) + ? MLK_ERR_INVALID_SK + : 0; + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(test, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL || + return_value == MLK_ERR_PCT_FAIL) +); + +#if defined(MLK_CONFIG_KEYGEN_PCT) +/* Specification: + * Partially implements 'Pairwise Consistency Test' @[FIPS140_3_IG, p.87] and + * @[FIPS203, Section 7.1, Pairwise Consistency]. */ + +/* Reference: Not implemented in the reference implementation @[REF]. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_PCT_FAIL The consistency check failed. */ +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context); + MLK_ALLOC(ss_enc, uint8_t, MLKEM_SSBYTES, context); + MLK_ALLOC(ss_dec, uint8_t, MLKEM_SSBYTES, context); + + if (ct == NULL || ss_enc == NULL || ss_dec == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + ret = mlk_kem_enc(ct, ss_enc, pk, context); + if (ret != 0) + { + goto cleanup; + } + + ret = mlk_kem_dec(ss_dec, ct, sk, context); + if (ret != 0) + { + goto cleanup; + } + +#if defined(MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST) + /* Deliberately break PCT for testing purposes */ + if (mlk_break_pct()) + { + ss_enc[0] = ~ss_enc[0]; + } +#endif /* MLK_CONFIG_KEYGEN_PCT_BREAKAGE_TEST */ + + ret = mlk_ct_memcmp(ss_enc, ss_dec, MLKEM_SSBYTES); + /* The result of the PCT is public. */ + MLK_CT_TESTING_DECLASSIFY(&ret, sizeof(ret)); + + if (ret != 0) + { + ret = MLK_ERR_PCT_FAIL; + } + +cleanup: + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(ss_dec, uint8_t, MLKEM_SSBYTES, context); + MLK_FREE(ss_enc, uint8_t, MLKEM_SSBYTES, context); + MLK_FREE(ct, uint8_t, MLKEM_INDCCA_CIPHERTEXTBYTES, context); + + /* The key pair being tested was just generated by this library, so a key + * check rejecting it hints at a faulty implementation rather than at bad + * input. Report it as a PCT failure. */ + if (ret == MLK_ERR_INVALID_PK || ret == MLK_ERR_INVALID_SK) + { + ret = MLK_ERR_PCT_FAIL; + } + + /* Other error codes, e.g. platform failures like out of memory or + * randomness failure, are passed on unmodified. */ + + return ret; +} +#else /* MLK_CONFIG_KEYGEN_PCT */ +MLK_MUST_CHECK_RETURN_VALUE +static int mlk_check_pct(uint8_t const pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t const sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + /* Skip PCT */ + ((void)pk); + ((void)sk); + MLK_CONTEXT_UNUSED(context); + return 0; +} +#endif /* !MLK_CONFIG_KEYGEN_PCT */ + +/* Reference: `crypto_kem_keypair_derand()` in the reference implementation + * @[REF]. + * - We optionally include PCT which is not present in + * the reference code. */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + const uint8_t coins[2 * MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret; + + ret = mlk_indcpa_keypair_derand(pk, sk, coins, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(sk + MLKEM_INDCPA_SECRETKEYBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_hash_h(sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, pk, + MLKEM_INDCCA_PUBLICKEYBYTES); + /* Value z for pseudo-random output on reject */ + mlk_memcpy(sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES, + coins + MLKEM_SYMBYTES, MLKEM_SYMBYTES); + + /* Declassify public key */ + MLK_CT_TESTING_DECLASSIFY(pk, MLKEM_INDCCA_PUBLICKEYBYTES); + + /* Pairwise Consistency Test (PCT) @[FIPS140_3_IG, p.87] */ + ret = mlk_check_pct(pk, sk, context); + if (ret != 0) + { + goto cleanup; + } + +cleanup: + if (ret != 0) + { + mlk_zeroize(pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_zeroize(sk, MLKEM_INDCCA_SECRETKEYBYTES); + } + + return ret; +} + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/* Reference: `crypto_kem_keypair()` in the reference implementation @[REF] + * - We zeroize the stack buffer */ +MLK_EXTERNAL_API +int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(coins, uint8_t, 2 * MLKEM_SYMBYTES, context); + + if (coins == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Acquire necessary randomness, and mark it as secret. */ + if (mlk_randombytes(coins, 2 * MLKEM_SYMBYTES) != 0) + { + ret = MLK_ERR_RNG_FAIL; + goto cleanup; + } + + MLK_CT_TESTING_SECRET(coins, 2 * MLKEM_SYMBYTES); + + ret = mlk_kem_keypair_derand(pk, sk, coins, context); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(coins, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: `crypto_kem_enc_derand()` in the reference implementation @[REF] + * - We include public key check + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + + if (buf == NULL || kr == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Specification: Implements @[FIPS203, Section 7.2, Modulus check] */ + ret = mlk_kem_check_pk(pk, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(buf, coins, MLKEM_SYMBYTES); + + /* Multitarget countermeasure for coins + contributory KEM */ + mlk_hash_h(buf + MLKEM_SYMBYTES, pk, MLKEM_INDCCA_PUBLICKEYBYTES); + mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES); + + /* coins are in kr+MLKEM_SYMBYTES */ + ret = mlk_indcpa_enc(ct, buf, pk, kr + MLKEM_SYMBYTES, context); + if (ret != 0) + { + goto cleanup; + } + + mlk_memcpy(ss, kr, MLKEM_SYMBYTES); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + return ret; +} + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/* Reference: `crypto_kem_enc()` in the reference implementation @[REF] + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + MLK_ALLOC(coins, uint8_t, MLKEM_SYMBYTES, context); + + if (coins == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + if (mlk_randombytes(coins, MLKEM_SYMBYTES) != 0) + { + ret = MLK_ERR_RNG_FAIL; + goto cleanup; + } + + MLK_CT_TESTING_SECRET(coins, MLKEM_SYMBYTES); + + ret = mlk_kem_enc_derand(ct, ss, pk, coins, context); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(coins, uint8_t, MLKEM_SYMBYTES, context); + return ret; +} +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `crypto_kem_dec()` in the reference implementation @[REF] + * - We include secret key check + * - We include stack buffer zeroization */ +MLK_EXTERNAL_API +int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES], + const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +{ + int ret = 0; + uint8_t fail; + const uint8_t *pk = sk + MLKEM_INDCPA_SECRETKEYBYTES; + MLK_ALLOC(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_ALLOC(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES, + context); + + if (buf == NULL || kr == NULL || tmp == NULL) + { + ret = MLK_ERR_OUT_OF_MEMORY; + goto cleanup; + } + + /* Specification: Implements @[FIPS203, Section 7.3, Hash check] */ + ret = mlk_kem_check_sk(sk, context); + if (ret != 0) + { + goto cleanup; + } + + ret = mlk_indcpa_dec(buf, ct, sk, context); + if (ret != 0) + { + goto cleanup; + } + + /* Multitarget countermeasure for coins + contributory KEM */ + mlk_memcpy(buf + MLKEM_SYMBYTES, + sk + MLKEM_INDCCA_SECRETKEYBYTES - 2 * MLKEM_SYMBYTES, + MLKEM_SYMBYTES); + mlk_hash_g(kr, buf, 2 * MLKEM_SYMBYTES); + + /* Recompute and compare ciphertext */ + /* coins are in kr+MLKEM_SYMBYTES */ + ret = mlk_indcpa_enc(tmp, buf, pk, kr + MLKEM_SYMBYTES, context); + if (ret != 0) + { + goto cleanup; + } + + fail = mlk_ct_memcmp(ct, tmp, MLKEM_INDCCA_CIPHERTEXTBYTES); + + /* Compute rejection key */ + mlk_memcpy(tmp, sk + MLKEM_INDCCA_SECRETKEYBYTES - MLKEM_SYMBYTES, + MLKEM_SYMBYTES); + mlk_memcpy(tmp + MLKEM_SYMBYTES, ct, MLKEM_INDCCA_CIPHERTEXTBYTES); + mlk_hash_j(ss, tmp, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES); + + /* Copy true key to return buffer if fail is 0 */ + mlk_ct_cmov_zero(ss, kr, MLKEM_SYMBYTES, fail); + +cleanup: + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + MLK_FREE(tmp, uint8_t, MLKEM_SYMBYTES + MLKEM_INDCCA_CIPHERTEXTBYTES, + context); + MLK_FREE(kr, uint8_t, 2 * MLKEM_SYMBYTES, context); + MLK_FREE(buf, uint8_t, 2 * MLKEM_SYMBYTES, context); + + return ret; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_check_pct diff --git a/src/mlkem-native/mlkem/src/kem.h b/src/mlkem-native/mlkem/src/kem.h new file mode 100644 index 0000000000..2de960c9c0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/kem.h @@ -0,0 +1,353 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#ifndef MLK_KEM_H +#define MLK_KEM_H + +#include "cbmc.h" +#include "common.h" +#include "sys.h" + +#if defined(MLK_CHECK_APIS) +/* Include to ensure consistency between internal kem.h + * and external mlkem_native.h. */ +#include "mlkem_native.h" + +#if MLKEM_INDCCA_SECRETKEYBYTES != \ + MLKEM_SECRETKEYBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for SECRETKEYBYTES between kem.h and mlkem_native.h +#endif + +#if MLKEM_INDCCA_PUBLICKEYBYTES != \ + MLKEM_PUBLICKEYBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for PUBLICKEYBYTES between kem.h and mlkem_native.h +#endif + +#if MLKEM_INDCCA_CIPHERTEXTBYTES != \ + MLKEM_CIPHERTEXTBYTES(MLK_CONFIG_PARAMETER_SET) +#error Mismatch for CIPHERTEXTBYTES between kem.h and mlkem_native.h +#endif + +#endif /* MLK_CHECK_APIS */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_kem_keypair_derand \ + MLK_NAMESPACE_K(keypair_derand) MLK_CONTEXT_PARAMETERS_3 +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +#define mlk_kem_keypair MLK_NAMESPACE_K(keypair) MLK_CONTEXT_PARAMETERS_2 +#endif +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +#define mlk_kem_enc_derand MLK_NAMESPACE_K(enc_derand) MLK_CONTEXT_PARAMETERS_4 +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +#define mlk_kem_enc MLK_NAMESPACE_K(enc) MLK_CONTEXT_PARAMETERS_3 +#endif +#define mlk_kem_check_pk MLK_NAMESPACE_K(check_pk) MLK_CONTEXT_PARAMETERS_1 +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_kem_dec MLK_NAMESPACE_K(dec) MLK_CONTEXT_PARAMETERS_3 +#define mlk_kem_check_sk MLK_NAMESPACE_K(check_sk) MLK_CONTEXT_PARAMETERS_1 +#endif + +/** + * Implements modulus check mandated by FIPS 203, i.e., ensures that + * coefficients are in [0,q-1]. + * + * @spec{Implements @[FIPS203, Section 7.2, 'modulus check'].} + * + * @reference{Not implemented in the reference implementation @[REF].} + * + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK Modulus check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_pk(const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_PK || + return_value == MLK_ERR_OUT_OF_MEMORY) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + + +/** + * Implements public key hash check mandated by FIPS 203, i.e., ensures that + * sk[768𝑘+32 ∶ 768𝑘+64] = H(pk) = H(sk[384𝑘 : 768𝑘+32]). + * + * @spec{Implements @[FIPS203, Section 7.3, 'hash check'].} + * + * @reference{Not implemented in the reference implementation @[REF].} + * + * @param[in] sk Input private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK Public key hash check failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_check_sk(const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_SK || + return_value == MLK_ERR_OUT_OF_MEMORY) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 16, ML-KEM.KeyGen_Internal].} + * + * @param[out] pk Output public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param[in] coins Input randomness (an already allocated array filled + * with 2*MLKEM_SYMBYTES random bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL MLK_CONFIG_KEYGEN_PCT enabled and random + * number generation failed within the PCT. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair_derand(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + const uint8_t coins[2 * MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + requires(memory_no_alias(coins, 2 * MLKEM_SYMBYTES)) + assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_PCT_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(sk, MLKEM_INDCCA_SECRETKEYBYTES)) +); + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate a public/private keypair for the ML-KEM key encapsulation mechanism. + * + * @spec{Implements @[FIPS203, Algorithm 19, ML-KEM.KeyGen].} + * + * @param[out] pk Output public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[out] sk Output private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_PCT_FAIL MLK_CONFIG_KEYGEN_PCT enabled and PCT failed. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_keypair(uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + assigns(memory_slice(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_PCT_FAIL || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(sk, MLKEM_INDCCA_SECRETKEYBYTES)) +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 17, ML-KEM.Encaps_Internal].} + * + * @param[out] ct Output ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param[in] coins Input randomness (an already allocated array filled + * with MLKEM_SYMBYTES random bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_enc_derand(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + const uint8_t coins[MLKEM_SYMBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + requires(memory_no_alias(coins, MLKEM_SYMBYTES)) + assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_PK || + return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ss, MLKEM_SSBYTES)) +); + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +/** + * Generate ciphertext and shared secret for a given public key. + * + * @spec{Implements @[FIPS203, Algorithm 20, ML-KEM.Encaps].} + * + * @param[out] ct Output ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] pk Input public key (an already allocated array of + * MLKEM_INDCCA_PUBLICKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + * @retval MLK_ERR_RNG_FAIL Random number generation failed. + * @retval MLK_ERR_INVALID_PK The 'modulus check' @[FIPS203, Section 7.2] + * for the public key failed. + */ +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_enc(uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + uint8_t ss[MLKEM_SSBYTES], + const uint8_t pk[MLKEM_INDCCA_PUBLICKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(pk, MLKEM_INDCCA_PUBLICKEYBYTES)) + assigns(memory_slice(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_PK || + return_value == MLK_ERR_OUT_OF_MEMORY || + return_value == MLK_ERR_RNG_FAIL) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ss, MLKEM_SSBYTES)) +); +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API */ + +/** + * Generate shared secret for a given ciphertext and private key. + * + * @spec{Implements @[FIPS203, Algorithm 21, ML-KEM.Decaps].} + * + * @param[out] ss Output shared secret (an already allocated array of + * MLKEM_SSBYTES bytes). + * @param[in] ct Input ciphertext (an already allocated array of + * MLKEM_INDCCA_CIPHERTEXTBYTES bytes). + * @param[in] sk Input private key (an already allocated array of + * MLKEM_INDCCA_SECRETKEYBYTES bytes). + * @param context Application context. Only present when + * MLK_CONFIG_CONTEXT_PARAMETER is defined; type set by + * MLK_CONFIG_CONTEXT_PARAMETER_TYPE. + * + * @retval 0 Success. + * @retval MLK_ERR_INVALID_SK The 'hash check' @[FIPS203, Section 7.3] + * for the secret key failed. + * @retval MLK_ERR_OUT_OF_MEMORY MLK_CONFIG_CUSTOM_ALLOC_FREE was used and + * MLK_CUSTOM_ALLOC returned NULL. + */ +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_EXTERNAL_API +MLK_MUST_CHECK_RETURN_VALUE +int mlk_kem_dec(uint8_t ss[MLKEM_SSBYTES], + const uint8_t ct[MLKEM_INDCCA_CIPHERTEXTBYTES], + const uint8_t sk[MLKEM_INDCCA_SECRETKEYBYTES], + MLK_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__( + requires(memory_no_alias(ss, MLKEM_SSBYTES)) + requires(memory_no_alias(ct, MLKEM_INDCCA_CIPHERTEXTBYTES)) + requires(memory_no_alias(sk, MLKEM_INDCCA_SECRETKEYBYTES)) + assigns(memory_slice(ss, MLKEM_SSBYTES)) + ensures(return_value == 0 || return_value == MLK_ERR_INVALID_SK || + return_value == MLK_ERR_OUT_OF_MEMORY) + /* Output buffers on error, per API-CONVENTIONS.md */ + ensures(return_value != 0 ==> + array_unchanged_or_zeroized_u8(ss, MLKEM_SSBYTES)) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_KEM_H */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/README.md b/src/mlkem-native/mlkem/src/native/aarch64/README.md new file mode 100644 index 0000000000..0d56871d6a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/README.md @@ -0,0 +1,16 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# AArch64 backend (little endian) + +This directory contains a native backend for little endian AArch64 systems. It is derived from [^NeonNTT] [^SLOTHY_Paper]. + +The code in this directory is auto-generated from the 'clean' assembly in [dev/aarch64_clean](../../../../dev/aarch64_clean) +in a two-step fashion: First, it is superoptimized using the [SLOTHY](https://github.com/slothy-optimizer/slothy) superoptimizer, +giving the assembly in [dev/aarch64_opt](../../../../dev/aarch64_opt). Then, it is stripped of remaining register aliases, macros +and most preprocessor directives by [`scripts/simpasm`](../../../../scripts/simpasm). + +If you want to understand how the assembly works, and/or make changes to it, consult [dev/](../../../../dev). + + +[^NeonNTT]: Becker, Hwang, Kannwischer, Yang, Yang: Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1, [https://eprint.iacr.org/2021/986](https://eprint.iacr.org/2021/986) +[^SLOTHY_Paper]: Abdulrahman, Becker, Kannwischer, Klein: Fast and Clean: Auditable high-performance assembly via constraint solving, [https://eprint.iacr.org/2022/1303](https://eprint.iacr.org/2022/1303) diff --git a/src/mlkem-native/mlkem/src/native/aarch64/meta.h b/src/mlkem-native/mlkem/src/native/aarch64/meta.h new file mode 100644 index 0000000000..6e8b9dc9e8 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/meta.h @@ -0,0 +1,166 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_AARCH64_META_H +#define MLK_NATIVE_AARCH64_META_H + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#define MLK_USE_NATIVE_POLY_TOBYTES +#define MLK_USE_NATIVE_REJ_UNIFORM + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_AARCH64 + + +#if !defined(__ASSEMBLER__) +#include "../api.h" +#include "src/arith_native_aarch64.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_ntt_aarch64_asm(data, mlk_aarch64_ntt_zetas_layer12345, + mlk_aarch64_ntt_zetas_layer67); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_intt_aarch64_asm(data, mlk_aarch64_invntt_zetas_layer12345, + mlk_aarch64_invntt_zetas_layer67); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_reduce_aarch64_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_tomont_aarch64_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_mulcache_compute_aarch64_asm( + x, y, mlk_aarch64_zetas_mulcache_native, + mlk_aarch64_zetas_mulcache_twisted_native); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + mlk_poly_tobytes_aarch64_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON) || len != MLKEM_N || + buflen % 24 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + return (int)mlk_rej_uniform_aarch64_asm(r, buf, buflen, + mlk_rej_uniform_table); +} +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_AARCH64_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c b/src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c new file mode 100644 index 0000000000..c743d92f18 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/aarch64_zetas.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_aarch64.h" + +/* + * Table of zeta values used in the AArch64 forward NTT + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_ntt_zetas_layer12345[80] = { + -1600, -15749, -749, -7373, -40, -394, -687, -6762, 630, 6201, + -1432, -14095, 848, 8347, 0, 0, 1062, 10453, 296, 2914, + -882, -8682, 0, 0, -1410, -13879, 1339, 13180, 1476, 14529, + 0, 0, 193, 1900, -283, -2786, 56, 551, 0, 0, + 797, 7845, -1089, -10719, 1333, 13121, 0, 0, -543, -5345, + 1426, 14036, -1235, -12156, 0, 0, -69, -679, 535, 5266, + -447, -4400, 0, 0, 569, 5601, -936, -9213, -450, -4429, + 0, 0, -1583, -15582, -1355, -13338, 821, 8081, 0, 0, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_ntt_zetas_layer67[384] = { + 289, 289, 331, 331, -76, -76, -1573, -1573, 2845, + 2845, 3258, 3258, -748, -748, -15483, -15483, 17, 17, + 583, 583, 1637, 1637, -1041, -1041, 167, 167, 5739, + 5739, 16113, 16113, -10247, -10247, -568, -568, -680, -680, + 723, 723, 1100, 1100, -5591, -5591, -6693, -6693, 7117, + 7117, 10828, 10828, 1197, 1197, -1025, -1025, -1052, -1052, + -1274, -1274, 11782, 11782, -10089, -10089, -10355, -10355, -12540, + -12540, 1409, 1409, -48, -48, 756, 756, -314, -314, + 13869, 13869, -472, -472, 7441, 7441, -3091, -3091, -667, + -667, 233, 233, -1173, -1173, -279, -279, -6565, -6565, + 2293, 2293, -11546, -11546, -2746, -2746, 650, 650, -1352, + -1352, -816, -816, 632, 632, 6398, 6398, -13308, -13308, + -8032, -8032, 6221, 6221, -1626, -1626, -540, -540, -1482, + -1482, 1461, 1461, -16005, -16005, -5315, -5315, -14588, -14588, + 14381, 14381, 1651, 1651, -1540, -1540, 952, 952, -642, + -642, 16251, 16251, -15159, -15159, 9371, 9371, -6319, -6319, + -464, -464, 33, 33, 1320, 1320, -1414, -1414, -4567, + -4567, 325, 325, 12993, 12993, -13918, -13918, 939, 939, + -892, -892, 733, 733, 268, 268, 9243, 9243, -8780, + -8780, 7215, 7215, 2638, 2638, -1021, -1021, -941, -941, + -992, -992, 641, 641, -10050, -10050, -9262, -9262, -9764, + -9764, 6309, 6309, -1010, -1010, 1435, 1435, 807, 807, + 452, 452, -9942, -9942, 14125, 14125, 7943, 7943, 4449, + 4449, 1584, 1584, -1292, -1292, 375, 375, -1239, -1239, + 15592, 15592, -12717, -12717, 3691, 3691, -12196, -12196, -1031, + -1031, -109, -109, -780, -780, 1645, 1645, -10148, -10148, + -1073, -1073, -7678, -7678, 16192, 16192, 1438, 1438, -461, + -461, 1534, 1534, -927, -927, 14155, 14155, -4538, -4538, + 15099, 15099, -9125, -9125, 1063, 1063, -556, -556, -1230, + -1230, -863, -863, 10463, 10463, -5473, -5473, -12107, -12107, + -8495, -8495, 319, 319, 757, 757, 561, 561, -735, + -735, 3140, 3140, 7451, 7451, 5522, 5522, -7235, -7235, + -682, -682, -712, -712, 1481, 1481, 648, 648, -6713, + -6713, -7008, -7008, 14578, 14578, 6378, 6378, -525, -525, + 403, 403, 1143, 1143, -554, -554, -5168, -5168, 3967, + 3967, 11251, 11251, -5453, -5453, 1092, 1092, 1026, 1026, + -1179, -1179, 886, 886, 10749, 10749, 10099, 10099, -11605, + -11605, 8721, 8721, -855, -855, -219, -219, 1227, 1227, + 910, 910, -8416, -8416, -2156, -2156, 12078, 12078, 8957, + 8957, -1607, -1607, -1455, -1455, -1219, -1219, 885, 885, + -15818, -15818, -14322, -14322, -11999, -11999, 8711, 8711, 1212, + 1212, 1029, 1029, -394, -394, -1175, -1175, 11930, 11930, + 10129, 10129, -3878, -3878, -11566, -11566, +}; + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_invntt_zetas_layer12345[80] = { + 1583, 15582, -821, -8081, 1355, 13338, 0, 0, -569, + -5601, 450, 4429, 936, 9213, 0, 0, 69, 679, + 447, 4400, -535, -5266, 0, 0, 543, 5345, 1235, + 12156, -1426, -14036, 0, 0, -797, -7845, -1333, -13121, + 1089, 10719, 0, 0, -193, -1900, -56, -551, 283, + 2786, 0, 0, 1410, 13879, -1476, -14529, -1339, -13180, + 0, 0, -1062, -10453, 882, 8682, -296, -2914, 0, + 0, 1600, 15749, 40, 394, 749, 7373, -848, -8347, + 1432, 14095, -630, -6201, 687, 6762, 0, 0, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_invntt_zetas_layer67[384] = { + -910, -910, -1227, -1227, 219, 219, 855, 855, -8957, + -8957, -12078, -12078, 2156, 2156, 8416, 8416, 1175, 1175, + 394, 394, -1029, -1029, -1212, -1212, 11566, 11566, 3878, + 3878, -10129, -10129, -11930, -11930, -885, -885, 1219, 1219, + 1455, 1455, 1607, 1607, -8711, -8711, 11999, 11999, 14322, + 14322, 15818, 15818, -648, -648, -1481, -1481, 712, 712, + 682, 682, -6378, -6378, -14578, -14578, 7008, 7008, 6713, + 6713, -886, -886, 1179, 1179, -1026, -1026, -1092, -1092, + -8721, -8721, 11605, 11605, -10099, -10099, -10749, -10749, 554, + 554, -1143, -1143, -403, -403, 525, 525, 5453, 5453, + -11251, -11251, -3967, -3967, 5168, 5168, 927, 927, -1534, + -1534, 461, 461, -1438, -1438, 9125, 9125, -15099, -15099, + 4538, 4538, -14155, -14155, 735, 735, -561, -561, -757, + -757, -319, -319, 7235, 7235, -5522, -5522, -7451, -7451, + -3140, -3140, 863, 863, 1230, 1230, 556, 556, -1063, + -1063, 8495, 8495, 12107, 12107, 5473, 5473, -10463, -10463, + -452, -452, -807, -807, -1435, -1435, 1010, 1010, -4449, + -4449, -7943, -7943, -14125, -14125, 9942, 9942, -1645, -1645, + 780, 780, 109, 109, 1031, 1031, -16192, -16192, 7678, + 7678, 1073, 1073, 10148, 10148, 1239, 1239, -375, -375, + 1292, 1292, -1584, -1584, 12196, 12196, -3691, -3691, 12717, + 12717, -15592, -15592, 1414, 1414, -1320, -1320, -33, -33, + 464, 464, 13918, 13918, -12993, -12993, -325, -325, 4567, + 4567, -641, -641, 992, 992, 941, 941, 1021, 1021, + -6309, -6309, 9764, 9764, 9262, 9262, 10050, 10050, -268, + -268, -733, -733, 892, 892, -939, -939, -2638, -2638, + -7215, -7215, 8780, 8780, -9243, -9243, -632, -632, 816, + 816, 1352, 1352, -650, -650, -6221, -6221, 8032, 8032, + 13308, 13308, -6398, -6398, 642, 642, -952, -952, 1540, + 1540, -1651, -1651, 6319, 6319, -9371, -9371, 15159, 15159, + -16251, -16251, -1461, -1461, 1482, 1482, 540, 540, 1626, + 1626, -14381, -14381, 14588, 14588, 5315, 5315, 16005, 16005, + 1274, 1274, 1052, 1052, 1025, 1025, -1197, -1197, 12540, + 12540, 10355, 10355, 10089, 10089, -11782, -11782, 279, 279, + 1173, 1173, -233, -233, 667, 667, 2746, 2746, 11546, + 11546, -2293, -2293, 6565, 6565, 314, 314, -756, -756, + 48, 48, -1409, -1409, 3091, 3091, -7441, -7441, 472, + 472, -13869, -13869, 1573, 1573, 76, 76, -331, -331, + -289, -289, 15483, 15483, 748, 748, -3258, -3258, -2845, + -2845, -1100, -1100, -723, -723, 680, 680, 568, 568, + -10828, -10828, -7117, -7117, 6693, 6693, 5591, 5591, 1041, + 1041, -1637, -1637, -583, -583, -17, -17, 10247, 10247, + -16113, -16113, -5739, -5739, -167, -167, +}; +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_zetas_mulcache_native[128] = { + 17, -17, -568, 568, 583, -583, -680, 680, 1637, -1637, + 723, -723, -1041, 1041, 1100, -1100, 1409, -1409, -667, 667, + -48, 48, 233, -233, 756, -756, -1173, 1173, -314, 314, + -279, 279, -1626, 1626, 1651, -1651, -540, 540, -1540, 1540, + -1482, 1482, 952, -952, 1461, -1461, -642, 642, 939, -939, + -1021, 1021, -892, 892, -941, 941, 733, -733, -992, 992, + 268, -268, 641, -641, 1584, -1584, -1031, 1031, -1292, 1292, + -109, 109, 375, -375, -780, 780, -1239, 1239, 1645, -1645, + 1063, -1063, 319, -319, -556, 556, 757, -757, -1230, 1230, + 561, -561, -863, 863, -735, 735, -525, 525, 1092, -1092, + 403, -403, 1026, -1026, 1143, -1143, -1179, 1179, -554, 554, + 886, -886, -1607, 1607, 1212, -1212, -1455, 1455, 1029, -1029, + -1219, 1219, -394, 394, 885, -885, -1175, 1175, +}; + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t + mlk_aarch64_zetas_mulcache_twisted_native[128] = { + 167, -167, -5591, 5591, 5739, -5739, -6693, 6693, 16113, + -16113, 7117, -7117, -10247, 10247, 10828, -10828, 13869, -13869, + -6565, 6565, -472, 472, 2293, -2293, 7441, -7441, -11546, + 11546, -3091, 3091, -2746, 2746, -16005, 16005, 16251, -16251, + -5315, 5315, -15159, 15159, -14588, 14588, 9371, -9371, 14381, + -14381, -6319, 6319, 9243, -9243, -10050, 10050, -8780, 8780, + -9262, 9262, 7215, -7215, -9764, 9764, 2638, -2638, 6309, + -6309, 15592, -15592, -10148, 10148, -12717, 12717, -1073, 1073, + 3691, -3691, -7678, 7678, -12196, 12196, 16192, -16192, 10463, + -10463, 3140, -3140, -5473, 5473, 7451, -7451, -12107, 12107, + 5522, -5522, -8495, 8495, -7235, 7235, -5168, 5168, 10749, + -10749, 3967, -3967, 10099, -10099, 11251, -11251, -11605, 11605, + -5453, 5453, 8721, -8721, -15818, 15818, 11930, -11930, -14322, + 14322, 10129, -10129, -11999, 11999, -3878, 3878, 8711, -8711, + -11566, 11566, +}; + +#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(aarch64_zetas) + +#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h b/src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h new file mode 100644 index 0000000000..d7c3d48e5f --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/arith_native_aarch64.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H +#define MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H + +#include "../../../cbmc.h" +#include "../../../common.h" + +#define mlk_aarch64_ntt_zetas_layer12345 \ + MLK_NAMESPACE(aarch64_ntt_zetas_layer12345) +#define mlk_aarch64_ntt_zetas_layer67 MLK_NAMESPACE(aarch64_ntt_zetas_layer67) +#define mlk_aarch64_invntt_zetas_layer12345 \ + MLK_NAMESPACE(aarch64_invntt_zetas_layer12345) +#define mlk_aarch64_invntt_zetas_layer67 \ + MLK_NAMESPACE(aarch64_invntt_zetas_layer67) +#define mlk_aarch64_zetas_mulcache_native \ + MLK_NAMESPACE(aarch64_zetas_mulcache_native) +#define mlk_aarch64_zetas_mulcache_twisted_native \ + MLK_NAMESPACE(aarch64_zetas_mulcache_twisted_native) +#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table) + +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_ntt_zetas_layer12345[80]; +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_aarch64_ntt_zetas_layer67[384]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_invntt_zetas_layer12345[80]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_invntt_zetas_layer67[384]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_zetas_mulcache_native[128]; +MLK_INTERNAL_DATA_DECLARATION const int16_t + mlk_aarch64_zetas_mulcache_twisted_native[128]; +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096]; + +#define mlk_ntt_aarch64_asm MLK_NAMESPACE(ntt_aarch64_asm) +void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], + const int16_t twiddles56[384]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_ntt_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(p, 0, MLKEM_N, 8192)) + requires(twiddles12345 == mlk_aarch64_ntt_zetas_layer12345) + requires(twiddles56 == mlk_aarch64_ntt_zetas_layer67) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(p, 0, MLKEM_N, 23595)) + /* check-magic: on */ +); + +#define mlk_intt_aarch64_asm MLK_NAMESPACE(intt_aarch64_asm) +void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], + const int16_t twiddles56[384]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_intt_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(twiddles12345 == mlk_aarch64_invntt_zetas_layer12345) + requires(twiddles56 == mlk_aarch64_invntt_zetas_layer67) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(p, 0, MLKEM_N, 26625)) + /* check-magic: on */ +); + +#define mlk_poly_reduce_aarch64_asm MLK_NAMESPACE(poly_reduce_aarch64_asm) +void mlk_poly_reduce_aarch64_asm(int16_t p[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_reduce_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_tomont_aarch64_asm MLK_NAMESPACE(poly_tomont_aarch64_asm) +void mlk_poly_tomont_aarch64_asm(int16_t p[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_tomont_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) +); + +#define mlk_poly_mulcache_compute_aarch64_asm \ + MLK_NAMESPACE(poly_mulcache_compute_aarch64_asm) +void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128], + const int16_t mlk_poly[256], + const int16_t zetas[128], + const int16_t zetas_twisted[128]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_mulcache_compute_aarch64_asm.ml + */ +__contract__( + requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N)) + requires(zetas == mlk_aarch64_zetas_mulcache_native) + requires(zetas_twisted == mlk_aarch64_zetas_mulcache_twisted_native) + assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q)) +); + +#define mlk_poly_tobytes_aarch64_asm MLK_NAMESPACE(poly_tobytes_aarch64_asm) +void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_poly_tobytes_aarch64_asm.ml */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm( + int16_t r[256], const int16_t a[512], const int16_t b[512], + const int16_t b_cache[256]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm( + int16_t r[256], const int16_t a[768], const int16_t b[768], + const int16_t b_cache[384]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) +void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm( + int16_t r[256], const int16_t a[1024], const int16_t b[1024], + const int16_t b_cache[512]) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/aarch64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_rej_uniform_aarch64_asm MLK_NAMESPACE(rej_uniform_aarch64_asm) +MLK_MUST_CHECK_RETURN_VALUE +uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf, + unsigned buflen, const uint8_t table[4096]) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/aarch64/proofs/mlkem_rej_uniform_aarch64_asm.ml. */ +__contract__( + requires(buflen % 24 == 0) + requires(memory_no_alias(buf, buflen)) + requires(table == mlk_rej_uniform_table) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value <= MLKEM_N) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); + +#endif /* !MLK_NATIVE_AARCH64_SRC_ARITH_NATIVE_AARCH64_H */ diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S new file mode 100644 index 0000000000..0901957893 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_intt_aarch64_asm.S @@ -0,0 +1,635 @@ +/* Copyright (c) 2022 Arm Limited + * Copyright (c) 2022 Hanno Becker + * Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [SLOTHY_Paper] + * Fast and Clean: Auditable high-performance assembly via constraint solving + * Abdulrahman, Becker, Kannwischer, Klein + * https://eprint.iacr.org/2022/1303 + */ + +/*yaml + Name: intt_aarch64_asm + Description: AArch64 ML-KEM inverse NTT following @[NeonNTT] and @[SLOTHY_Paper] + Signature: void mlk_intt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + x1: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const int16_t twiddles12345[80] + description: Twiddle factors for layers 1-5 + x2: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t twiddles56[384] + description: Twiddle factors for layers 6-7 + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_intt_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(intt_aarch64_asm) +MLK_ASM_FN_SYMBOL(intt_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w5, #0xd01 // =3329 + mov v7.h[0], w5 + mov w5, #0x4ebf // =20159 + mov v7.h[1], w5 + mov w5, #0x200 // =512 + dup v29.8h, w5 + mov w5, #0x13b0 // =5040 + dup v30.8h, w5 + mov x3, x0 + mov x4, #0x8 // =8 + ldr q13, [x3, #0x20] + ldr q8, [x3, #0x30] + ldr q6, [x3] + ldr q16, [x3, #0x10] + ldr q4, [x3, #0x50] + ldr q11, [x3, #0x40] + ldr q3, [x3, #0x70] + trn1 v23.4s, v13.4s, v8.4s + ldr q0, [x3, #0x60] + trn2 v19.4s, v6.4s, v16.4s + trn2 v21.4s, v13.4s, v8.4s + trn1 v6.4s, v6.4s, v16.4s + ldr q24, [x2, #0x20] + trn1 v10.2d, v19.2d, v21.2d + ldr q16, [x2], #0x60 + trn1 v5.2d, v6.2d, v23.2d + trn1 v28.4s, v0.4s, v3.4s + trn2 v18.2d, v6.2d, v23.2d + mul v31.8h, v10.8h, v29.8h + trn2 v13.4s, v0.4s, v3.4s + ldur q14, [x2, #-0x50] + sqrdmulh v26.8h, v18.8h, v30.8h + ldur q20, [x2, #-0x20] + mul v17.8h, v18.8h, v29.8h + trn2 v18.2d, v19.2d, v21.2d + mul v9.8h, v18.8h, v29.8h + trn1 v12.4s, v11.4s, v4.4s + sqrdmulh v22.8h, v18.8h, v30.8h + sqrdmulh v3.8h, v10.8h, v30.8h + sqrdmulh v25.8h, v5.8h, v30.8h + mls v9.8h, v22.8h, v7.h[0] + mls v17.8h, v26.8h, v7.h[0] + trn2 v26.4s, v11.4s, v4.4s + mul v8.8h, v5.8h, v29.8h + trn1 v10.2d, v26.2d, v13.2d + ldur q11, [x2, #-0x10] + mls v31.8h, v3.8h, v7.h[0] + trn1 v6.2d, v12.2d, v28.2d + trn2 v3.2d, v26.2d, v13.2d + ldur q4, [x2, #-0x30] + mls v8.8h, v25.8h, v7.h[0] + sub v19.8h, v17.8h, v9.8h + trn2 v13.2d, v12.2d, v28.2d + sqrdmulh v1.8h, v3.8h, v30.8h + add v9.8h, v17.8h, v9.8h + mul v18.8h, v19.8h, v20.8h + add v28.8h, v8.8h, v31.8h + sqrdmulh v20.8h, v19.8h, v11.8h + sub v12.8h, v28.8h, v9.8h + sub v23.8h, v8.8h, v31.8h + sqrdmulh v11.8h, v13.8h, v30.8h + sqrdmulh v5.8h, v23.8h, v4.8h + mul v0.8h, v23.8h, v24.8h + mul v2.8h, v13.8h, v29.8h + mls v0.8h, v5.8h, v7.h[0] + add v24.8h, v28.8h, v9.8h + mls v18.8h, v20.8h, v7.h[0] + sqrdmulh v15.8h, v6.8h, v30.8h + sqrdmulh v25.8h, v12.8h, v14.8h + mul v21.8h, v12.8h, v16.8h + sub v23.8h, v0.8h, v18.8h + sqrdmulh v8.8h, v23.8h, v14.8h + mul v23.8h, v23.8h, v16.8h + mls v21.8h, v25.8h, v7.h[0] + mls v23.8h, v8.8h, v7.h[0] + mul v14.8h, v3.8h, v29.8h + add v3.8h, v0.8h, v18.8h + trn2 v4.4s, v24.4s, v3.4s + mls v14.8h, v1.8h, v7.h[0] + trn1 v9.4s, v24.4s, v3.4s + trn2 v12.4s, v21.4s, v23.4s + mls v2.8h, v11.8h, v7.h[0] + trn1 v28.4s, v21.4s, v23.4s + ldr q11, [x1], #0x10 + mul v31.8h, v10.8h, v29.8h + trn1 v25.2d, v4.2d, v12.2d + trn1 v20.2d, v9.2d, v28.2d + ldr q23, [x2, #0x50] + trn2 v13.2d, v4.2d, v12.2d + sqrdmulh v21.8h, v10.8h, v30.8h + trn2 v4.2d, v9.2d, v28.2d + ldr q9, [x2, #0x40] + mul v27.8h, v6.8h, v29.8h + add v26.8h, v20.8h, v25.8h + sub v3.8h, v2.8h, v14.8h + sqdmulh v12.8h, v26.8h, v7.h[1] + add v5.8h, v4.8h, v13.8h + sub v8.8h, v4.8h, v13.8h + add v10.8h, v2.8h, v14.8h + sqdmulh v6.8h, v5.8h, v7.h[1] + ldr q2, [x2, #0x10] + mls v27.8h, v15.8h, v7.h[0] + ldr q15, [x2, #0x20] + srshr v12.8h, v12.8h, #0xb + mls v31.8h, v21.8h, v7.h[0] + srshr v6.8h, v6.8h, #0xb + sqrdmulh v23.8h, v3.8h, v23.8h + mls v26.8h, v12.8h, v7.h[0] + add v21.8h, v27.8h, v31.8h + mls v5.8h, v6.8h, v7.h[0] + sub v6.8h, v27.8h, v31.8h + sub v14.8h, v21.8h, v10.8h + ldr q27, [x2], #0x60 + mul v3.8h, v3.8h, v9.8h + mls v3.8h, v23.8h, v7.h[0] + ldur q13, [x2, #-0x30] + sub v12.8h, v26.8h, v5.8h + add v5.8h, v26.8h, v5.8h + sqrdmulh v31.8h, v8.8h, v11.h[5] + sqrdmulh v19.8h, v12.8h, v11.h[1] + mul v24.8h, v12.8h, v11.h[0] + sqrdmulh v13.8h, v6.8h, v13.8h + mls v24.8h, v19.8h, v7.h[0] + sub x4, x4, #0x2 + +Lmlk_intt_layer4567_start: + add v16.8h, v21.8h, v10.8h + mul v18.8h, v6.8h, v15.8h + sub v19.8h, v20.8h, v25.8h + ldr q21, [x3, #0xa0] + str q5, [x3], #0x40 + mls v18.8h, v13.8h, v7.h[0] + sqrdmulh v15.8h, v14.8h, v2.8h + ldr q10, [x3, #0x50] + ldr q12, [x3, #0x40] + stur q24, [x3, #-0x20] + mul v5.8h, v8.8h, v11.h[4] + sub v0.8h, v18.8h, v3.8h + ldr q24, [x3, #0x70] + mls v5.8h, v31.8h, v7.h[0] + ldr q26, [x2, #0x50] + trn2 v1.4s, v12.4s, v10.4s + add v6.8h, v18.8h, v3.8h + sqrdmulh v20.8h, v0.8h, v2.8h + trn1 v13.4s, v12.4s, v10.4s + trn1 v18.4s, v16.4s, v6.4s + mul v22.8h, v0.8h, v27.8h + trn1 v17.4s, v21.4s, v24.4s + sqrdmulh v0.8h, v19.8h, v11.h[3] + trn1 v25.2d, v13.2d, v17.2d + mls v22.8h, v20.8h, v7.h[0] + trn2 v21.4s, v21.4s, v24.4s + mul v24.8h, v25.8h, v29.8h + trn2 v28.2d, v13.2d, v17.2d + sqrdmulh v4.8h, v25.8h, v30.8h + trn2 v3.2d, v1.2d, v21.2d + mul v17.8h, v28.8h, v29.8h + sqrdmulh v31.8h, v28.8h, v30.8h + ldr q2, [x2, #0x10] + mls v24.8h, v4.8h, v7.h[0] + mul v4.8h, v19.8h, v11.h[2] + ldr q19, [x2, #0x40] + mls v4.8h, v0.8h, v7.h[0] + mul v0.8h, v14.8h, v27.8h + mls v0.8h, v15.8h, v7.h[0] + sub v8.8h, v4.8h, v5.8h + mul v12.8h, v3.8h, v29.8h + mul v23.8h, v8.8h, v11.h[0] + trn2 v28.4s, v16.4s, v6.4s + sqrdmulh v10.8h, v8.8h, v11.h[1] + trn1 v9.4s, v0.4s, v22.4s + trn2 v22.4s, v0.4s, v22.4s + ldr q11, [x1], #0x10 + mls v17.8h, v31.8h, v7.h[0] + trn1 v20.2d, v18.2d, v9.2d + trn2 v14.2d, v18.2d, v9.2d + ldr q15, [x2, #0x20] + trn1 v6.2d, v1.2d, v21.2d + sqrdmulh v9.8h, v3.8h, v30.8h + trn1 v25.2d, v28.2d, v22.2d + trn2 v16.2d, v28.2d, v22.2d + mls v23.8h, v10.8h, v7.h[0] + add v1.8h, v20.8h, v25.8h + sqrdmulh v21.8h, v6.8h, v30.8h + add v8.8h, v14.8h, v16.8h + ldr q27, [x2], #0x60 + sqdmulh v28.8h, v8.8h, v7.h[1] + mls v12.8h, v9.8h, v7.h[0] + sqdmulh v31.8h, v1.8h, v7.h[1] + mul v0.8h, v6.8h, v29.8h + sub v10.8h, v17.8h, v12.8h + mls v0.8h, v21.8h, v7.h[0] + srshr v21.8h, v28.8h, #0xb + srshr v13.8h, v31.8h, #0xb + sqrdmulh v22.8h, v10.8h, v26.8h + mls v8.8h, v21.8h, v7.h[0] + mls v1.8h, v13.8h, v7.h[0] + add v21.8h, v24.8h, v0.8h + stur q23, [x3, #-0x10] + sub v6.8h, v24.8h, v0.8h + mul v3.8h, v10.8h, v19.8h + add v0.8h, v4.8h, v5.8h + sqdmulh v13.8h, v0.8h, v7.h[1] + ldur q10, [x2, #-0x30] + add v5.8h, v1.8h, v8.8h + mls v3.8h, v22.8h, v7.h[0] + sub v8.8h, v1.8h, v8.8h + mul v24.8h, v8.8h, v11.h[0] + sqrdmulh v8.8h, v8.8h, v11.h[1] + srshr v1.8h, v13.8h, #0xb + sqrdmulh v13.8h, v6.8h, v10.8h + mls v0.8h, v1.8h, v7.h[0] + add v10.8h, v17.8h, v12.8h + mls v24.8h, v8.8h, v7.h[0] + sub v8.8h, v14.8h, v16.8h + sqrdmulh v31.8h, v8.8h, v11.h[5] + sub v14.8h, v21.8h, v10.8h + stur q0, [x3, #-0x30] + sub x4, x4, #0x1 + cbnz x4, Lmlk_intt_layer4567_start + mul v15.8h, v6.8h, v15.8h + sub v22.8h, v20.8h, v25.8h + add v4.8h, v21.8h, v10.8h + str q24, [x3, #0x20] + mls v15.8h, v13.8h, v7.h[0] + str q5, [x3], #0x40 + ldr q9, [x1], #0x10 + sqrdmulh v28.8h, v14.8h, v2.8h + mul v16.8h, v14.8h, v27.8h + sub v18.8h, v15.8h, v3.8h + add v15.8h, v15.8h, v3.8h + sqrdmulh v0.8h, v18.8h, v2.8h + trn2 v24.4s, v4.4s, v15.4s + trn1 v2.4s, v4.4s, v15.4s + mul v18.8h, v18.8h, v27.8h + mls v16.8h, v28.8h, v7.h[0] + mls v18.8h, v0.8h, v7.h[0] + mul v23.8h, v8.8h, v11.h[4] + sqrdmulh v12.8h, v22.8h, v11.h[3] + trn1 v17.4s, v16.4s, v18.4s + trn2 v4.4s, v16.4s, v18.4s + mls v23.8h, v31.8h, v7.h[0] + trn2 v3.2d, v2.2d, v17.2d + trn2 v6.2d, v24.2d, v4.2d + mul v26.8h, v22.8h, v11.h[2] + trn1 v28.2d, v2.2d, v17.2d + mls v26.8h, v12.8h, v7.h[0] + add v25.8h, v3.8h, v6.8h + sub v18.8h, v3.8h, v6.8h + trn1 v24.2d, v24.2d, v4.2d + sqdmulh v1.8h, v25.8h, v7.h[1] + sub v27.8h, v28.8h, v24.8h + sqrdmulh v2.8h, v18.8h, v9.h[5] + add v28.8h, v28.8h, v24.8h + mul v24.8h, v27.8h, v9.h[2] + sqdmulh v12.8h, v28.8h, v7.h[1] + mul v20.8h, v18.8h, v9.h[4] + mls v20.8h, v2.8h, v7.h[0] + srshr v1.8h, v1.8h, #0xb + sqrdmulh v19.8h, v27.8h, v9.h[3] + srshr v15.8h, v12.8h, #0xb + mls v25.8h, v1.8h, v7.h[0] + add v8.8h, v26.8h, v23.8h + sub v4.8h, v26.8h, v23.8h + mls v28.8h, v15.8h, v7.h[0] + mls v24.8h, v19.8h, v7.h[0] + mul v2.8h, v4.8h, v11.h[0] + sub v19.8h, v28.8h, v25.8h + sqrdmulh v15.8h, v4.8h, v11.h[1] + add v25.8h, v28.8h, v25.8h + sub v10.8h, v24.8h, v20.8h + str q25, [x3], #0x40 + sqrdmulh v22.8h, v19.8h, v9.h[1] + add v28.8h, v24.8h, v20.8h + sqrdmulh v25.8h, v10.8h, v9.h[1] + mul v27.8h, v19.8h, v9.h[0] + mul v26.8h, v10.8h, v9.h[0] + sqdmulh v20.8h, v28.8h, v7.h[1] + sqdmulh v16.8h, v8.8h, v7.h[1] + mls v26.8h, v25.8h, v7.h[0] + mls v2.8h, v15.8h, v7.h[0] + srshr v15.8h, v20.8h, #0xb + srshr v1.8h, v16.8h, #0xb + mls v27.8h, v22.8h, v7.h[0] + mls v28.8h, v15.8h, v7.h[0] + mls v8.8h, v1.8h, v7.h[0] + stur q27, [x3, #-0x20] + stur q2, [x3, #-0x50] + stur q28, [x3, #-0x30] + stur q26, [x3, #-0x10] + stur q8, [x3, #-0x70] + mov x4, #0x4 // =4 + ldr q0, [x1], #0x20 + ldur q1, [x1, #-0x10] + ldr q26, [x0] + ldr q13, [x0, #0x40] + ldr q28, [x0, #0xc0] + ldr q2, [x0, #0x140] + ldr q6, [x0, #0x80] + ldr q9, [x0, #0x100] + ldr q29, [x0, #0x1c0] + ldr q23, [x0, #0x180] + sub v17.8h, v26.8h, v13.8h + add v4.8h, v26.8h, v13.8h + ldr q25, [x0, #0xd0] + ldr q24, [x0, #0x50] + add v5.8h, v6.8h, v28.8h + mul v19.8h, v17.8h, v0.h[6] + sub v10.8h, v6.8h, v28.8h + ldr q30, [x0, #0x150] + sqrdmulh v12.8h, v17.8h, v0.h[7] + add v17.8h, v9.8h, v2.8h + sub v28.8h, v9.8h, v2.8h + ldr q2, [x0, #0x90] + sub v26.8h, v23.8h, v29.8h + sqrdmulh v31.8h, v10.8h, v1.h[1] + add v22.8h, v23.8h, v29.8h + ldr q3, [x0, #0x110] + sqrdmulh v9.8h, v28.8h, v1.h[3] + sub v20.8h, v4.8h, v5.8h + sub v27.8h, v17.8h, v22.8h + ldr q29, [x0, #0x10] + add v16.8h, v4.8h, v5.8h + sqrdmulh v4.8h, v26.8h, v1.h[5] + add v6.8h, v17.8h, v22.8h + ldr q22, [x0, #0x1d0] + mul v8.8h, v28.8h, v1.h[2] + sub v21.8h, v2.8h, v25.8h + sub v5.8h, v16.8h, v6.8h + mul v17.8h, v26.8h, v1.h[4] + mul v26.8h, v10.8h, v1.h[0] + mls v26.8h, v31.8h, v7.h[0] + mls v17.8h, v4.8h, v7.h[0] + mls v19.8h, v12.8h, v7.h[0] + mls v8.8h, v9.8h, v7.h[0] + sqrdmulh v10.8h, v27.8h, v0.h[5] + sub v12.8h, v19.8h, v26.8h + add v9.8h, v19.8h, v26.8h + sqrdmulh v26.8h, v20.8h, v0.h[3] + sub v11.8h, v8.8h, v17.8h + add v14.8h, v8.8h, v17.8h + sqrdmulh v13.8h, v12.8h, v0.h[3] + add v23.8h, v9.8h, v14.8h + sqrdmulh v28.8h, v11.8h, v0.h[5] + sub v19.8h, v9.8h, v14.8h + mul v17.8h, v27.8h, v0.h[4] + str q23, [x0, #0x40] + mul v14.8h, v20.8h, v0.h[2] + mul v8.8h, v11.8h, v0.h[4] + mul v4.8h, v12.8h, v0.h[2] + mls v14.8h, v26.8h, v7.h[0] + mls v17.8h, v10.8h, v7.h[0] + mls v8.8h, v28.8h, v7.h[0] + mls v4.8h, v13.8h, v7.h[0] + sub v10.8h, v14.8h, v17.8h + add v20.8h, v14.8h, v17.8h + sqrdmulh v28.8h, v5.8h, v0.h[1] + mul v18.8h, v5.8h, v0.h[0] + str q20, [x0, #0x80] + sub v13.8h, v4.8h, v8.8h + mul v23.8h, v10.8h, v0.h[0] + mul v17.8h, v19.8h, v0.h[0] + sqrdmulh v9.8h, v13.8h, v0.h[1] + mls v18.8h, v28.8h, v7.h[0] + sqrdmulh v10.8h, v10.8h, v0.h[1] + sub x4, x4, #0x2 + +Lmlk_intt_layer123_start: + sub v12.8h, v3.8h, v30.8h + mul v11.8h, v21.8h, v1.h[0] + add v28.8h, v4.8h, v8.8h + ldr q20, [x0, #0x190] + add v27.8h, v16.8h, v6.8h + sqrdmulh v8.8h, v12.8h, v1.h[3] + add v16.8h, v29.8h, v24.8h + str q28, [x0, #0xc0] + mls v23.8h, v10.8h, v7.h[0] + str q27, [x0], #0x10 + add v15.8h, v20.8h, v22.8h + str q18, [x0, #0xf0] + mul v14.8h, v13.8h, v0.h[0] + add v2.8h, v2.8h, v25.8h + sub v26.8h, v20.8h, v22.8h + mul v4.8h, v12.8h, v1.h[2] + sub v5.8h, v16.8h, v2.8h + str q23, [x0, #0x170] + add v20.8h, v3.8h, v30.8h + sqrdmulh v27.8h, v26.8h, v1.h[5] + add v16.8h, v16.8h, v2.8h + mul v18.8h, v26.8h, v1.h[4] + sub v31.8h, v20.8h, v15.8h + mls v4.8h, v8.8h, v7.h[0] + sub v28.8h, v29.8h, v24.8h + mls v18.8h, v27.8h, v7.h[0] + ldr q22, [x0, #0x1d0] + mul v26.8h, v28.8h, v0.h[6] + mul v2.8h, v5.8h, v0.h[2] + sub v12.8h, v4.8h, v18.8h + sqrdmulh v24.8h, v28.8h, v0.h[7] + mls v14.8h, v9.8h, v7.h[0] + sqrdmulh v10.8h, v12.8h, v0.h[5] + mls v26.8h, v24.8h, v7.h[0] + ldr q24, [x0, #0x50] + mul v8.8h, v12.8h, v0.h[4] + str q14, [x0, #0x1b0] + add v28.8h, v4.8h, v18.8h + sqrdmulh v5.8h, v5.8h, v0.h[3] + add v6.8h, v20.8h, v15.8h + sqrdmulh v3.8h, v19.8h, v0.h[1] + sub v13.8h, v16.8h, v6.8h + sqrdmulh v12.8h, v21.8h, v1.h[1] + sqrdmulh v21.8h, v13.8h, v0.h[1] + sqrdmulh v27.8h, v31.8h, v0.h[5] + ldr q25, [x0, #0xd0] + mls v11.8h, v12.8h, v7.h[0] + mul v23.8h, v31.8h, v0.h[4] + mul v18.8h, v13.8h, v0.h[0] + add v30.8h, v26.8h, v11.8h + sub v13.8h, v26.8h, v11.8h + mls v23.8h, v27.8h, v7.h[0] + add v12.8h, v30.8h, v28.8h + sub v19.8h, v30.8h, v28.8h + mls v2.8h, v5.8h, v7.h[0] + str q12, [x0, #0x40] + sqrdmulh v26.8h, v13.8h, v0.h[3] + mls v8.8h, v10.8h, v7.h[0] + ldr q30, [x0, #0x150] + sub v20.8h, v2.8h, v23.8h + mul v4.8h, v13.8h, v0.h[2] + add v13.8h, v2.8h, v23.8h + mls v4.8h, v26.8h, v7.h[0] + ldr q2, [x0, #0x90] + mul v23.8h, v20.8h, v0.h[0] + ldr q29, [x0, #0x10] + sqrdmulh v10.8h, v20.8h, v0.h[1] + str q13, [x0, #0x80] + sub v13.8h, v4.8h, v8.8h + mls v17.8h, v3.8h, v7.h[0] + ldr q3, [x0, #0x110] + mls v18.8h, v21.8h, v7.h[0] + sub v21.8h, v2.8h, v25.8h + sqrdmulh v9.8h, v13.8h, v0.h[1] + str q17, [x0, #0x130] + mul v17.8h, v19.8h, v0.h[0] + sub x4, x4, #0x1 + cbnz x4, Lmlk_intt_layer123_start + mls v23.8h, v10.8h, v7.h[0] + ldr q11, [x0, #0x190] + str q18, [x0, #0x100] + add v27.8h, v3.8h, v30.8h + mul v13.8h, v13.8h, v0.h[0] + sub v5.8h, v29.8h, v24.8h + add v14.8h, v16.8h, v6.8h + mls v13.8h, v9.8h, v7.h[0] + add v10.8h, v11.8h, v22.8h + str q23, [x0, #0x180] + sub v20.8h, v11.8h, v22.8h + sub v23.8h, v27.8h, v10.8h + sqrdmulh v16.8h, v21.8h, v1.h[1] + sqrdmulh v31.8h, v23.8h, v0.h[5] + str q13, [x0, #0x1c0] + add v13.8h, v4.8h, v8.8h + mul v18.8h, v21.8h, v1.h[0] + str q13, [x0, #0xc0] + sqrdmulh v13.8h, v19.8h, v0.h[1] + sqrdmulh v28.8h, v20.8h, v1.h[5] + str q14, [x0], #0x10 + mul v4.8h, v20.8h, v1.h[4] + mls v17.8h, v13.8h, v7.h[0] + sub v13.8h, v3.8h, v30.8h + sqrdmulh v8.8h, v13.8h, v1.h[3] + mul v12.8h, v13.8h, v1.h[2] + mls v4.8h, v28.8h, v7.h[0] + mls v12.8h, v8.8h, v7.h[0] + mls v18.8h, v16.8h, v7.h[0] + str q17, [x0, #0x130] + sqrdmulh v15.8h, v5.8h, v0.h[7] + add v11.8h, v27.8h, v10.8h + mul v16.8h, v5.8h, v0.h[6] + sub v8.8h, v12.8h, v4.8h + sqrdmulh v28.8h, v8.8h, v0.h[5] + add v13.8h, v2.8h, v25.8h + mls v16.8h, v15.8h, v7.h[0] + add v26.8h, v12.8h, v4.8h + mul v8.8h, v8.8h, v0.h[4] + add v4.8h, v29.8h, v24.8h + mls v8.8h, v28.8h, v7.h[0] + sub v20.8h, v4.8h, v13.8h + add v14.8h, v4.8h, v13.8h + add v12.8h, v16.8h, v18.8h + sqrdmulh v22.8h, v20.8h, v0.h[3] + add v27.8h, v14.8h, v11.8h + sub v13.8h, v16.8h, v18.8h + mul v4.8h, v20.8h, v0.h[2] + str q27, [x0], #0x10 + sub v24.8h, v12.8h, v26.8h + sqrdmulh v3.8h, v13.8h, v0.h[3] + mul v13.8h, v13.8h, v0.h[2] + sqrdmulh v27.8h, v24.8h, v0.h[1] + mls v13.8h, v3.8h, v7.h[0] + mul v9.8h, v24.8h, v0.h[0] + mls v9.8h, v27.8h, v7.h[0] + add v30.8h, v13.8h, v8.8h + sub v13.8h, v13.8h, v8.8h + mls v4.8h, v22.8h, v7.h[0] + str q30, [x0, #0xb0] + sqrdmulh v16.8h, v13.8h, v0.h[1] + str q9, [x0, #0x130] + mul v9.8h, v13.8h, v0.h[0] + add v13.8h, v12.8h, v26.8h + str q13, [x0, #0x30] + mul v13.8h, v23.8h, v0.h[4] + sub v23.8h, v14.8h, v11.8h + mls v13.8h, v31.8h, v7.h[0] + mls v9.8h, v16.8h, v7.h[0] + mul v30.8h, v23.8h, v0.h[0] + sub v24.8h, v4.8h, v13.8h + add v13.8h, v4.8h, v13.8h + sqrdmulh v23.8h, v23.8h, v0.h[1] + str q9, [x0, #0x1b0] + str q13, [x0, #0x70] + sqrdmulh v13.8h, v24.8h, v0.h[1] + mul v21.8h, v24.8h, v0.h[0] + mls v30.8h, v23.8h, v7.h[0] + mls v21.8h, v13.8h, v7.h[0] + str q30, [x0, #0xf0] + str q21, [x0, #0x170] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(intt_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S new file mode 100644 index 0000000000..2383c0f05e --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_ntt_aarch64_asm.S @@ -0,0 +1,565 @@ +/* Copyright (c) 2022 Arm Limited + * Copyright (c) 2022 Hanno Becker + * Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [SLOTHY_Paper] + * Fast and Clean: Auditable high-performance assembly via constraint solving + * Abdulrahman, Becker, Kannwischer, Klein + * https://eprint.iacr.org/2022/1303 + */ + +/*yaml + Name: ntt_aarch64_asm + Description: AArch64 ML-KEM forward NTT following @[NeonNTT] and @[SLOTHY_Paper] + Signature: void mlk_ntt_aarch64_asm(int16_t p[256], const int16_t twiddles12345[80], const int16_t twiddles56[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + x1: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const int16_t twiddles12345[80] + description: Twiddle factors for layers 1-5 + x2: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t twiddles56[384] + description: Twiddle factors for layers 6-7 + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_ntt_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_aarch64_asm) +MLK_ASM_FN_SYMBOL(ntt_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w5, #0xd01 // =3329 + mov v7.h[0], w5 + mov w5, #0x4ebf // =20159 + mov v7.h[1], w5 + mov x3, x0 + mov x4, #0x4 // =4 + ldr q0, [x1], #0x20 + ldur q1, [x1, #-0x10] + ldr q21, [x0, #0x40] + ldr q5, [x0, #0x1c0] + ldr q30, [x0, #0x110] + ldr q24, [x0, #0x140] + ldr q12, [x0, #0x80] + sqrdmulh v9.8h, v5.8h, v0.h[1] + mul v23.8h, v5.8h, v0.h[0] + sqrdmulh v17.8h, v24.8h, v0.h[1] + ldr q13, [x0, #0xc0] + mls v23.8h, v9.8h, v7.h[0] + mul v8.8h, v24.8h, v0.h[0] + mls v8.8h, v17.8h, v7.h[0] + add v9.8h, v13.8h, v23.8h + sub v10.8h, v13.8h, v23.8h + mul v11.8h, v30.8h, v0.h[0] + ldr q13, [x0, #0x180] + sqrdmulh v28.8h, v9.8h, v0.h[3] + sub v29.8h, v21.8h, v8.8h + mul v26.8h, v9.8h, v0.h[2] + add v8.8h, v21.8h, v8.8h + mul v2.8h, v13.8h, v0.h[0] + mls v26.8h, v28.8h, v7.h[0] + mul v28.8h, v10.8h, v0.h[4] + sqrdmulh v23.8h, v10.8h, v0.h[5] + add v22.8h, v8.8h, v26.8h + sqrdmulh v10.8h, v13.8h, v0.h[1] + sqrdmulh v21.8h, v22.8h, v0.h[7] + ldr q13, [x0, #0x100] + mul v16.8h, v22.8h, v0.h[6] + mls v28.8h, v23.8h, v7.h[0] + mls v2.8h, v10.8h, v7.h[0] + sqrdmulh v23.8h, v13.8h, v0.h[1] + sub v10.8h, v29.8h, v28.8h + add v17.8h, v29.8h, v28.8h + mls v16.8h, v21.8h, v7.h[0] + sub v18.8h, v12.8h, v2.8h + ldr q29, [x0] + sqrdmulh v14.8h, v17.8h, v1.h[3] + add v22.8h, v12.8h, v2.8h + sqrdmulh v9.8h, v18.8h, v0.h[5] + mul v21.8h, v13.8h, v0.h[0] + ldr q13, [x0, #0x150] + mul v5.8h, v18.8h, v0.h[4] + mls v5.8h, v9.8h, v7.h[0] + mul v18.8h, v13.8h, v0.h[0] + mls v21.8h, v23.8h, v7.h[0] + sqrdmulh v2.8h, v13.8h, v0.h[1] + mul v13.8h, v17.8h, v1.h[2] + sub v4.8h, v29.8h, v21.8h + mls v13.8h, v14.8h, v7.h[0] + add v25.8h, v29.8h, v21.8h + add v6.8h, v4.8h, v5.8h + sqrdmulh v15.8h, v22.8h, v0.h[3] + sub v21.8h, v4.8h, v5.8h + sub v5.8h, v8.8h, v26.8h + mul v23.8h, v22.8h, v0.h[2] + add v28.8h, v6.8h, v13.8h + sub v13.8h, v6.8h, v13.8h + mul v4.8h, v5.8h, v1.h[0] + sub x4, x4, #0x2 + +Lmlk_ntt_layer123_start: + mls v23.8h, v15.8h, v7.h[0] + ldr q6, [x0, #0x190] + ldr q15, [x0, #0x90] + ldr q19, [x0, #0x10] + mul v22.8h, v10.8h, v1.h[4] + ldr q24, [x0, #0x50] + str q13, [x0, #0x140] + sqrdmulh v13.8h, v6.8h, v0.h[1] + sub v20.8h, v25.8h, v23.8h + sqrdmulh v3.8h, v30.8h, v0.h[1] + str q28, [x0, #0x100] + ldr q30, [x0, #0x120] + mul v8.8h, v6.8h, v0.h[0] + sqrdmulh v27.8h, v10.8h, v1.h[5] + mls v11.8h, v3.8h, v7.h[0] + mls v18.8h, v2.8h, v7.h[0] + ldr q31, [x0, #0x160] + sqrdmulh v10.8h, v5.8h, v1.h[1] + mls v8.8h, v13.8h, v7.h[0] + ldr q13, [x0, #0x1d0] + sub v14.8h, v24.8h, v18.8h + add v9.8h, v24.8h, v18.8h + sqrdmulh v2.8h, v31.8h, v0.h[1] + mls v4.8h, v10.8h, v7.h[0] + add v10.8h, v25.8h, v23.8h + sub v24.8h, v19.8h, v11.8h + add v25.8h, v19.8h, v11.8h + sqrdmulh v28.8h, v13.8h, v0.h[1] + mul v11.8h, v30.8h, v0.h[0] + mul v17.8h, v13.8h, v0.h[0] + sub v13.8h, v10.8h, v16.8h + sub v6.8h, v15.8h, v8.8h + mls v17.8h, v28.8h, v7.h[0] + str q13, [x0, #0x40] + mls v22.8h, v27.8h, v7.h[0] + ldr q13, [x0, #0xd0] + add v26.8h, v20.8h, v4.8h + mul v18.8h, v31.8h, v0.h[0] + add v27.8h, v10.8h, v16.8h + str q26, [x0, #0x80] + sqrdmulh v31.8h, v6.8h, v0.h[5] + add v3.8h, v21.8h, v22.8h + str q27, [x0], #0x10 + mul v26.8h, v6.8h, v0.h[4] + add v6.8h, v13.8h, v17.8h + sub v5.8h, v13.8h, v17.8h + str q3, [x0, #0x170] + sub v17.8h, v21.8h, v22.8h + sqrdmulh v10.8h, v6.8h, v0.h[3] + sub v13.8h, v20.8h, v4.8h + add v20.8h, v15.8h, v8.8h + sqrdmulh v12.8h, v5.8h, v0.h[5] + str q13, [x0, #0xb0] + mul v8.8h, v6.8h, v0.h[2] + str q17, [x0, #0x1b0] + mls v8.8h, v10.8h, v7.h[0] + mul v29.8h, v5.8h, v0.h[4] + mls v29.8h, v12.8h, v7.h[0] + sub v5.8h, v9.8h, v8.8h + add v3.8h, v9.8h, v8.8h + sqrdmulh v15.8h, v20.8h, v0.h[3] + mul v4.8h, v5.8h, v1.h[0] + add v6.8h, v14.8h, v29.8h + sqrdmulh v9.8h, v3.8h, v0.h[7] + sqrdmulh v12.8h, v6.8h, v1.h[3] + sub v10.8h, v14.8h, v29.8h + mul v23.8h, v6.8h, v1.h[2] + mls v26.8h, v31.8h, v7.h[0] + mls v23.8h, v12.8h, v7.h[0] + mul v16.8h, v3.8h, v0.h[6] + add v13.8h, v24.8h, v26.8h + sub v21.8h, v24.8h, v26.8h + mls v16.8h, v9.8h, v7.h[0] + add v28.8h, v13.8h, v23.8h + sub v13.8h, v13.8h, v23.8h + mul v23.8h, v20.8h, v0.h[2] + sub x4, x4, #0x1 + cbnz x4, Lmlk_ntt_layer123_start + sqrdmulh v3.8h, v5.8h, v1.h[1] + mls v23.8h, v15.8h, v7.h[0] + ldr q5, [x0, #0x190] + mul v29.8h, v10.8h, v1.h[4] + mls v4.8h, v3.8h, v7.h[0] + sub v19.8h, v25.8h, v23.8h + sqrdmulh v31.8h, v5.8h, v0.h[1] + sqrdmulh v6.8h, v30.8h, v0.h[1] + sub v3.8h, v19.8h, v4.8h + mul v5.8h, v5.8h, v0.h[0] + str q3, [x0, #0xc0] + sqrdmulh v12.8h, v10.8h, v1.h[5] + mls v18.8h, v2.8h, v7.h[0] + ldr q3, [x0, #0x1d0] + mls v5.8h, v31.8h, v7.h[0] + sqrdmulh v10.8h, v3.8h, v0.h[1] + mls v11.8h, v6.8h, v7.h[0] + ldr q31, [x0, #0x90] + mul v30.8h, v3.8h, v0.h[0] + mls v30.8h, v10.8h, v7.h[0] + sub v10.8h, v31.8h, v5.8h + mls v29.8h, v12.8h, v7.h[0] + ldr q6, [x0, #0xd0] + sqrdmulh v15.8h, v10.8h, v0.h[5] + mul v17.8h, v10.8h, v0.h[4] + add v10.8h, v6.8h, v30.8h + sub v6.8h, v6.8h, v30.8h + sqrdmulh v12.8h, v10.8h, v0.h[3] + sub v27.8h, v21.8h, v29.8h + sqrdmulh v3.8h, v6.8h, v0.h[5] + mul v10.8h, v10.8h, v0.h[2] + ldr q20, [x0, #0x50] + mls v10.8h, v12.8h, v7.h[0] + mul v2.8h, v6.8h, v0.h[4] + add v6.8h, v20.8h, v18.8h + add v5.8h, v31.8h, v5.8h + mls v2.8h, v3.8h, v7.h[0] + sub v31.8h, v6.8h, v10.8h + sqrdmulh v12.8h, v5.8h, v0.h[3] + sub v22.8h, v20.8h, v18.8h + add v6.8h, v6.8h, v10.8h + mul v20.8h, v31.8h, v1.h[0] + add v30.8h, v22.8h, v2.8h + sqrdmulh v3.8h, v6.8h, v0.h[7] + sqrdmulh v10.8h, v30.8h, v1.h[3] + mul v9.8h, v30.8h, v1.h[2] + ldr q30, [x0, #0x10] + mls v17.8h, v15.8h, v7.h[0] + mls v9.8h, v10.8h, v7.h[0] + mul v15.8h, v6.8h, v0.h[6] + add v24.8h, v30.8h, v11.8h + sub v10.8h, v22.8h, v2.8h + mls v15.8h, v3.8h, v7.h[0] + add v6.8h, v19.8h, v4.8h + add v22.8h, v25.8h, v23.8h + sqrdmulh v3.8h, v10.8h, v1.h[5] + str q13, [x0, #0x140] + sub v19.8h, v30.8h, v11.8h + add v25.8h, v22.8h, v16.8h + mul v5.8h, v5.8h, v0.h[2] + sub v13.8h, v22.8h, v16.8h + str q28, [x0, #0x100] + mls v5.8h, v12.8h, v7.h[0] + str q13, [x0, #0x40] + str q6, [x0, #0x80] + add v21.8h, v21.8h, v29.8h + sqrdmulh v13.8h, v31.8h, v1.h[1] + str q25, [x0], #0x10 + add v12.8h, v19.8h, v17.8h + sub v31.8h, v19.8h, v17.8h + mul v30.8h, v10.8h, v1.h[4] + str q21, [x0, #0x170] + add v21.8h, v24.8h, v5.8h + add v6.8h, v12.8h, v9.8h + mls v30.8h, v3.8h, v7.h[0] + str q27, [x0, #0x1b0] + sub v10.8h, v21.8h, v15.8h + sub v12.8h, v12.8h, v9.8h + mls v20.8h, v13.8h, v7.h[0] + str q6, [x0, #0x100] + str q10, [x0, #0x40] + sub v13.8h, v24.8h, v5.8h + add v3.8h, v21.8h, v15.8h + str q12, [x0, #0x140] + sub v10.8h, v31.8h, v30.8h + add v21.8h, v31.8h, v30.8h + str q3, [x0], #0x10 + add v12.8h, v13.8h, v20.8h + sub v13.8h, v13.8h, v20.8h + str q21, [x0, #0x170] + str q10, [x0, #0x1b0] + str q12, [x0, #0x70] + str q13, [x0, #0xb0] + mov x0, x3 + mov x4, #0x8 // =8 + ldr q2, [x0, #0x20] + ldr q13, [x1], #0x10 + ldr q30, [x0, #0x30] + ldr q25, [x2, #0x40] + ldr q5, [x0] + ldr q18, [x0, #0x60] + ldr q12, [x0, #0x70] + sqrdmulh v17.8h, v2.8h, v13.h[1] + ldr q4, [x1], #0x10 + ldr q23, [x0, #0x10] + sqrdmulh v21.8h, v30.8h, v13.h[1] + ldr q24, [x2, #0x20] + ldr q9, [x2], #0x60 + mul v10.8h, v30.8h, v13.h[0] + mul v11.8h, v2.8h, v13.h[0] + mls v10.8h, v21.8h, v7.h[0] + sqrdmulh v29.8h, v12.8h, v4.h[1] + mul v1.8h, v12.8h, v4.h[0] + add v21.8h, v23.8h, v10.8h + sub v10.8h, v23.8h, v10.8h + mul v8.8h, v18.8h, v4.h[0] + sqrdmulh v23.8h, v21.8h, v13.h[3] + mul v2.8h, v21.8h, v13.h[2] + mls v1.8h, v29.8h, v7.h[0] + mls v2.8h, v23.8h, v7.h[0] + ldur q15, [x2, #-0x50] + sqrdmulh v0.8h, v10.8h, v13.h[5] + mls v11.8h, v17.8h, v7.h[0] + ldr q29, [x0, #0x50] + mul v23.8h, v10.8h, v13.h[4] + mls v23.8h, v0.8h, v7.h[0] + sub v16.8h, v29.8h, v1.8h + add v3.8h, v5.8h, v11.8h + sub v31.8h, v5.8h, v11.8h + sqrdmulh v22.8h, v16.8h, v4.h[5] + add v30.8h, v3.8h, v2.8h + sub v0.8h, v3.8h, v2.8h + sqrdmulh v28.8h, v18.8h, v4.h[1] + add v21.8h, v31.8h, v23.8h + sub v19.8h, v31.8h, v23.8h + mul v26.8h, v16.8h, v4.h[4] + trn2 v3.4s, v30.4s, v0.4s + ldur q23, [x2, #-0x10] + trn2 v18.4s, v21.4s, v19.4s + mls v26.8h, v22.8h, v7.h[0] + trn1 v13.4s, v30.4s, v0.4s + mls v8.8h, v28.8h, v7.h[0] + trn2 v31.2d, v3.2d, v18.2d + trn1 v11.4s, v21.4s, v19.4s + add v27.8h, v29.8h, v1.8h + sqrdmulh v6.8h, v31.8h, v15.8h + trn1 v2.2d, v13.2d, v11.2d + trn2 v13.2d, v13.2d, v11.2d + mul v1.8h, v31.8h, v9.8h + ldr q11, [x0, #0x40] + sqrdmulh v29.8h, v13.8h, v15.8h + mls v1.8h, v6.8h, v7.h[0] + trn1 v6.2d, v3.2d, v18.2d + mul v17.8h, v13.8h, v9.8h + sub v13.8h, v11.8h, v8.8h + sqrdmulh v10.8h, v27.8h, v4.h[3] + sub v12.8h, v13.8h, v26.8h + sub v18.8h, v6.8h, v1.8h + mls v17.8h, v29.8h, v7.h[0] + add v30.8h, v6.8h, v1.8h + add v6.8h, v13.8h, v26.8h + ldur q13, [x2, #-0x30] + sqrdmulh v16.8h, v18.8h, v23.8h + trn1 v28.4s, v6.4s, v12.4s + mul v23.8h, v18.8h, v25.8h + ldr q25, [x2, #0x10] + add v20.8h, v2.8h, v17.8h + mul v0.8h, v30.8h, v24.8h + sqrdmulh v29.8h, v30.8h, v13.8h + sub v30.8h, v2.8h, v17.8h + mls v23.8h, v16.8h, v7.h[0] + sub x4, x4, #0x2 + +Lmlk_ntt_layer4567_start: + ldr q19, [x2, #0x50] + sub v31.8h, v30.8h, v23.8h + mls v0.8h, v29.8h, v7.h[0] + add v16.8h, v11.8h, v8.8h + ldr q18, [x0, #0xa0] + trn2 v14.4s, v6.4s, v12.4s + mul v26.8h, v27.8h, v4.h[2] + ldr q4, [x1], #0x10 + ldr q24, [x2, #0x40] + ldr q21, [x0, #0xb0] + mls v26.8h, v10.8h, v7.h[0] + add v23.8h, v30.8h, v23.8h + sub v15.8h, v20.8h, v0.8h + ldr q9, [x0, #0x90] + add v10.8h, v20.8h, v0.8h + mul v8.8h, v18.8h, v4.h[0] + ldr q1, [x2], #0x60 + trn1 v27.4s, v23.4s, v31.4s + sqrdmulh v12.8h, v18.8h, v4.h[1] + trn1 v5.4s, v10.4s, v15.4s + sub v30.8h, v16.8h, v26.8h + trn2 v13.2d, v5.2d, v27.2d + sqrdmulh v2.8h, v21.8h, v4.h[1] + add v29.8h, v16.8h, v26.8h + mul v0.8h, v21.8h, v4.h[0] + str q13, [x0, #0x20] + trn1 v11.4s, v29.4s, v30.4s + mls v8.8h, v12.8h, v7.h[0] + trn2 v26.4s, v29.4s, v30.4s + trn2 v6.2d, v11.2d, v28.2d + mls v0.8h, v2.8h, v7.h[0] + trn2 v16.2d, v26.2d, v14.2d + trn1 v26.2d, v26.2d, v14.2d + trn1 v20.2d, v5.2d, v27.2d + sqrdmulh v29.8h, v6.8h, v25.8h + trn2 v15.4s, v10.4s, v15.4s + sqrdmulh v13.8h, v16.8h, v25.8h + str q20, [x0], #0x40 + sub v30.8h, v9.8h, v0.8h + add v27.8h, v9.8h, v0.8h + mul v17.8h, v6.8h, v1.8h + sqrdmulh v22.8h, v30.8h, v4.h[5] + mul v18.8h, v16.8h, v1.8h + mls v18.8h, v13.8h, v7.h[0] + mul v2.8h, v30.8h, v4.h[4] + mls v2.8h, v22.8h, v7.h[0] + trn2 v22.4s, v23.4s, v31.4s + sub v3.8h, v26.8h, v18.8h + ldur q25, [x2, #-0x30] + mls v17.8h, v29.8h, v7.h[0] + trn2 v31.2d, v15.2d, v22.2d + trn1 v20.2d, v15.2d, v22.2d + add v16.8h, v26.8h, v18.8h + sqrdmulh v26.8h, v3.8h, v19.8h + trn1 v21.2d, v11.2d, v28.2d + ldr q11, [x0, #0x40] + sqrdmulh v29.8h, v16.8h, v25.8h + stur q20, [x0, #-0x30] + add v20.8h, v21.8h, v17.8h + stur q31, [x0, #-0x10] + mul v23.8h, v3.8h, v24.8h + ldr q25, [x2, #0x10] + sub v13.8h, v11.8h, v8.8h + mls v23.8h, v26.8h, v7.h[0] + ldur q1, [x2, #-0x40] + sub v12.8h, v13.8h, v2.8h + add v6.8h, v13.8h, v2.8h + sqrdmulh v10.8h, v27.8h, v4.h[3] + sub v30.8h, v21.8h, v17.8h + mul v0.8h, v16.8h, v1.8h + trn1 v28.4s, v6.4s, v12.4s + sub x4, x4, #0x1 + cbnz x4, Lmlk_ntt_layer4567_start + add v22.8h, v11.8h, v8.8h + mul v27.8h, v27.8h, v4.h[2] + trn2 v17.4s, v6.4s, v12.4s + ldr q15, [x2], #0x60 + mls v27.8h, v10.8h, v7.h[0] + add v4.8h, v30.8h, v23.8h + sub v18.8h, v30.8h, v23.8h + ldur q6, [x2, #-0x30] + mls v0.8h, v29.8h, v7.h[0] + ldur q12, [x2, #-0x40] + ldur q24, [x2, #-0x20] + ldur q2, [x2, #-0x10] + trn1 v9.4s, v4.4s, v18.4s + add v10.8h, v22.8h, v27.8h + sub v13.8h, v22.8h, v27.8h + sub v1.8h, v20.8h, v0.8h + trn2 v21.4s, v10.4s, v13.4s + add v27.8h, v20.8h, v0.8h + trn2 v3.2d, v21.2d, v17.2d + trn1 v13.4s, v10.4s, v13.4s + trn1 v31.4s, v27.4s, v1.4s + sqrdmulh v10.8h, v3.8h, v25.8h + trn2 v5.2d, v13.2d, v28.2d + trn1 v13.2d, v13.2d, v28.2d + trn1 v21.2d, v21.2d, v17.2d + sqrdmulh v17.8h, v5.8h, v25.8h + trn2 v30.2d, v31.2d, v9.2d + mul v25.8h, v3.8h, v15.8h + str q30, [x0, #0x20] + trn2 v30.4s, v4.4s, v18.4s + mls v25.8h, v10.8h, v7.h[0] + trn2 v3.4s, v27.4s, v1.4s + mul v20.8h, v5.8h, v15.8h + trn2 v10.2d, v3.2d, v30.2d + mls v20.8h, v17.8h, v7.h[0] + str q10, [x0, #0x30] + sub v18.8h, v21.8h, v25.8h + add v10.8h, v21.8h, v25.8h + trn1 v3.2d, v3.2d, v30.2d + sqrdmulh v30.8h, v18.8h, v2.8h + mul v12.8h, v10.8h, v12.8h + sqrdmulh v6.8h, v10.8h, v6.8h + str q3, [x0, #0x10] + add v21.8h, v13.8h, v20.8h + mul v10.8h, v18.8h, v24.8h + sub v13.8h, v13.8h, v20.8h + mls v10.8h, v30.8h, v7.h[0] + mls v12.8h, v6.8h, v7.h[0] + trn1 v30.2d, v31.2d, v9.2d + sub v3.8h, v13.8h, v10.8h + add v6.8h, v13.8h, v10.8h + add v10.8h, v21.8h, v12.8h + sub v21.8h, v21.8h, v12.8h + trn2 v13.4s, v6.4s, v3.4s + trn1 v12.4s, v10.4s, v21.4s + trn2 v21.4s, v10.4s, v21.4s + trn1 v3.4s, v6.4s, v3.4s + str q30, [x0], #0x40 + trn2 v10.2d, v21.2d, v13.2d + trn1 v13.2d, v21.2d, v13.2d + trn2 v21.2d, v12.2d, v3.2d + trn1 v3.2d, v12.2d, v3.2d + str q10, [x0, #0x30] + str q13, [x0, #0x10] + str q3, [x0], #0x40 + stur q21, [x0, #-0x20] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(ntt_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S new file mode 100644 index 0000000000..4b869f8c92 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_mulcache_compute_aarch64_asm.S @@ -0,0 +1,130 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_mulcache_compute_aarch64_asm + Description: Compute multiplication cache for polynomial + Signature: void mlk_poly_mulcache_compute_aarch64_asm(int16_t cache[128], const int16_t mlk_poly[256], const int16_t zetas[128], const int16_t zetas_twisted[128]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 256 + permissions: write-only + c_parameter: int16_t cache[128] + description: Output cache + x1: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t mlk_poly[256] + description: Input polynomial + x2: + type: buffer + size_bytes: 256 + permissions: read-only + c_parameter: const int16_t zetas[128] + description: Zeta values + x3: + type: buffer + size_bytes: 256 + permissions: read-only + c_parameter: const int16_t zetas_twisted[128] + description: Twisted zeta values + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_mulcache_compute_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_mulcache_compute_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_mulcache_compute_aarch64_asm) + + .cfi_startproc + mov w5, #0xd01 // =3329 + dup v6.8h, w5 + mov w5, #0x4ebf // =20159 + dup v7.8h, w5 + mov x4, #0x10 // =16 + ldr q0, [x1], #0x20 + ldur q2, [x1, #-0x10] + ldr q19, [x1], #0x20 + ldr q29, [x3], #0x10 + ldur q16, [x1, #-0x10] + ldr q18, [x2], #0x10 + ldr q26, [x1], #0x20 + ldr q25, [x2], #0x10 + uzp2 v5.8h, v0.8h, v2.8h + ldr q28, [x3], #0x10 + ldur q7, [x1, #-0x10] + ldr q2, [x1], #0x20 + uzp2 v27.8h, v19.8h, v16.8h + sqrdmulh v16.8h, v5.8h, v29.8h + ldr q17, [x3], #0x10 + ldr q19, [x3], #0x10 + mul v5.8h, v5.8h, v18.8h + uzp2 v29.8h, v26.8h, v7.8h + mul v26.8h, v27.8h, v25.8h + sqrdmulh v4.8h, v27.8h, v28.8h + mls v5.8h, v16.8h, v6.h[0] + lsr x4, x4, #1 + sub x4, x4, #0x2 + +Lmlk_poly_mulcache_compute_loop_start: + str q5, [x0], #0x10 + sqrdmulh v22.8h, v29.8h, v17.8h + ldr q28, [x2], #0x10 + ldur q24, [x1, #-0x10] + ldr q0, [x1], #0x20 + mls v26.8h, v4.8h, v6.h[0] + ldur q16, [x1, #-0x10] + ldr q17, [x3], #0x10 + mul v5.8h, v29.8h, v28.8h + uzp2 v23.8h, v2.8h, v24.8h + ldr q18, [x2], #0x10 + mls v5.8h, v22.8h, v6.h[0] + uzp2 v29.8h, v0.8h, v16.8h + sqrdmulh v4.8h, v23.8h, v19.8h + ldr q2, [x1], #0x20 + ldr q19, [x3], #0x10 + str q26, [x0], #0x10 + mul v26.8h, v23.8h, v18.8h + subs x4, x4, #0x1 + cbnz x4, Lmlk_poly_mulcache_compute_loop_start + mls v26.8h, v4.8h, v6.h[0] + str q5, [x0], #0x10 + ldr q5, [x2], #0x10 + ldur q4, [x1, #-0x10] + sqrdmulh v16.8h, v29.8h, v17.8h + ldr q0, [x2], #0x10 + mul v29.8h, v29.8h, v5.8h + uzp2 v18.8h, v2.8h, v4.8h + str q26, [x0], #0x10 + sqrdmulh v17.8h, v18.8h, v19.8h + mls v29.8h, v16.8h, v6.h[0] + mul v26.8h, v18.8h, v0.8h + mls v26.8h, v17.8h, v6.h[0] + str q29, [x0], #0x10 + str q26, [x0], #0x10 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_mulcache_compute_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S new file mode 100644 index 0000000000..0244ac6af3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_reduce_aarch64_asm.S @@ -0,0 +1,153 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_reduce_aarch64_asm + Description: Barrett reduction of polynomial coefficients + Signature: void mlk_poly_reduce_aarch64_asm(int16_t p[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_reduce_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_reduce_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_reduce_aarch64_asm) + + .cfi_startproc + mov w2, #0xd01 // =3329 + dup v3.8h, w2 + mov w2, #0x4ebf // =20159 + dup v4.8h, w2 + mov x1, #0x8 // =8 + ldr q21, [x0], #0x40 + ldur q18, [x0, #-0x20] + ldur q0, [x0, #-0x30] + ldur q5, [x0, #-0x10] + ldr q26, [x0], #0x40 + sqdmulh v17.8h, v21.8h, v4.h[0] + sqdmulh v27.8h, v18.8h, v4.h[0] + sqdmulh v22.8h, v0.8h, v4.h[0] + srshr v17.8h, v17.8h, #0xb + sqdmulh v23.8h, v5.8h, v4.h[0] + srshr v29.8h, v27.8h, #0xb + mls v21.8h, v17.8h, v3.h[0] + srshr v17.8h, v22.8h, #0xb + mls v18.8h, v29.8h, v3.h[0] + srshr v22.8h, v23.8h, #0xb + mls v0.8h, v17.8h, v3.h[0] + sshr v2.8h, v21.8h, #0xf + mls v5.8h, v22.8h, v3.h[0] + sshr v29.8h, v18.8h, #0xf + and v19.16b, v3.16b, v2.16b + sqdmulh v2.8h, v26.8h, v4.h[0] + sshr v31.8h, v0.8h, #0xf + add v17.8h, v21.8h, v19.8h + and v21.16b, v3.16b, v29.16b + and v31.16b, v3.16b, v31.16b + sub x1, x1, #0x2 + +Lmlk_poly_reduce_loop_start: + add v21.8h, v18.8h, v21.8h + ldur q18, [x0, #-0x20] + add v25.8h, v0.8h, v31.8h + ldur q0, [x0, #-0x30] + stur q21, [x0, #-0x60] + sshr v28.8h, v5.8h, #0xf + stur q17, [x0, #-0x80] + srshr v23.8h, v2.8h, #0xb + sqdmulh v30.8h, v18.8h, v4.h[0] + stur q25, [x0, #-0x70] + and v22.16b, v3.16b, v28.16b + sqdmulh v7.8h, v0.8h, v4.h[0] + add v16.8h, v5.8h, v22.8h + ldur q5, [x0, #-0x10] + mls v26.8h, v23.8h, v3.h[0] + stur q16, [x0, #-0x50] + srshr v6.8h, v30.8h, #0xb + srshr v1.8h, v7.8h, #0xb + sqdmulh v19.8h, v5.8h, v4.h[0] + mls v18.8h, v6.8h, v3.h[0] + sshr v24.8h, v26.8h, #0xf + mls v0.8h, v1.8h, v3.h[0] + and v27.16b, v3.16b, v24.16b + srshr v29.8h, v19.8h, #0xb + add v17.8h, v26.8h, v27.8h + ldr q26, [x0], #0x40 + sshr v1.8h, v18.8h, #0xf + mls v5.8h, v29.8h, v3.h[0] + sshr v20.8h, v0.8h, #0xf + and v21.16b, v3.16b, v1.16b + and v31.16b, v3.16b, v20.16b + sqdmulh v2.8h, v26.8h, v4.h[0] + subs x1, x1, #0x1 + cbnz x1, Lmlk_poly_reduce_loop_start + add v28.8h, v0.8h, v31.8h + ldur q29, [x0, #-0x10] + add v21.8h, v18.8h, v21.8h + srshr v18.8h, v2.8h, #0xb + sshr v2.8h, v5.8h, #0xf + ldur q16, [x0, #-0x20] + stur q17, [x0, #-0x80] + ldur q0, [x0, #-0x30] + and v2.16b, v3.16b, v2.16b + sqdmulh v24.8h, v29.8h, v4.h[0] + stur q28, [x0, #-0x70] + stur q21, [x0, #-0x60] + add v31.8h, v5.8h, v2.8h + sqdmulh v6.8h, v16.8h, v4.h[0] + stur q31, [x0, #-0x50] + sqdmulh v17.8h, v0.8h, v4.h[0] + srshr v22.8h, v24.8h, #0xb + mls v26.8h, v18.8h, v3.h[0] + srshr v31.8h, v6.8h, #0xb + mls v29.8h, v22.8h, v3.h[0] + srshr v19.8h, v17.8h, #0xb + mls v16.8h, v31.8h, v3.h[0] + sshr v7.8h, v26.8h, #0xf + mls v0.8h, v19.8h, v3.h[0] + and v5.16b, v3.16b, v7.16b + sshr v22.8h, v29.8h, #0xf + add v27.8h, v26.8h, v5.8h + and v26.16b, v3.16b, v22.16b + sshr v20.8h, v16.8h, #0xf + stur q27, [x0, #-0x40] + and v2.16b, v3.16b, v20.16b + sshr v23.8h, v0.8h, #0xf + add v18.8h, v29.8h, v26.8h + add v31.8h, v16.8h, v2.8h + and v29.16b, v3.16b, v23.16b + stur q18, [x0, #-0x10] + add v25.8h, v0.8h, v29.8h + stur q31, [x0, #-0x20] + stur q25, [x0, #-0x30] + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_reduce_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S new file mode 100644 index 0000000000..57b798c171 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tobytes_aarch64_asm.S @@ -0,0 +1,124 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_tobytes_aarch64_asm + Description: Convert polynomial to byte representation + Signature: void mlk_poly_tobytes_aarch64_asm(uint8_t r[384], const int16_t a[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 384 + permissions: write-only + c_parameter: uint8_t r[384] + description: Output byte array + x1: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t a[256] + description: Input polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + !defined(MLK_CONFIG_NO_ENCAPS_API)) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_tobytes_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tobytes_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_tobytes_aarch64_asm) + + .cfi_startproc + mov x2, #0x10 // =16 + ldr q5, [x1, #0x10] + ldr q3, [x1], #0x20 + ldr q29, [x1], #0x20 + ldur q2, [x1, #-0x10] + ldr q27, [x1, #0x10] + ldr q23, [x1, #0x30] + ldr q17, [x1], #0x20 + ldr q16, [x1], #0x20 + uzp2 v26.8h, v3.8h, v5.8h + uzp1 v19.8h, v3.8h, v5.8h + uzp2 v0.8h, v29.8h, v2.8h + uzp1 v1.8h, v29.8h, v2.8h + xtn v5.8b, v26.8h + shrn v3.8b, v19.8h, #0x8 + shrn v4.8b, v26.8h, #0x4 + xtn v18.8b, v0.8h + shrn v30.8b, v0.8h, #0x4 + xtn v28.8b, v1.8h + shrn v29.8b, v1.8h, #0x8 + sli v3.8b, v5.8b, #0x4 + xtn v2.8b, v19.8h + sli v29.8b, v18.8b, #0x4 + lsr x2, x2, #1 + sub x2, x2, #0x2 + +Lmlk_poly_tobytes_loop_start: + uzp1 v25.8h, v17.8h, v27.8h + uzp2 v31.8h, v17.8h, v27.8h + uzp1 v24.8h, v16.8h, v23.8h + uzp2 v6.8h, v16.8h, v23.8h + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + shrn v3.8b, v25.8h, #0x8 + ldr q17, [x1], #0x20 + shrn v4.8b, v31.8h, #0x4 + xtn v21.8b, v6.8h + ldr q23, [x1, #0x10] + st3 { v28.8b, v29.8b, v30.8b }, [x0], #24 + shrn v29.8b, v24.8h, #0x8 + ldur q27, [x1, #-0x10] + xtn v20.8b, v31.8h + ldr q16, [x1], #0x20 + sli v29.8b, v21.8b, #0x4 + xtn v2.8b, v25.8h + sli v3.8b, v20.8b, #0x4 + xtn v28.8b, v24.8h + shrn v30.8b, v6.8h, #0x4 + subs x2, x2, #0x1 + cbnz x2, Lmlk_poly_tobytes_loop_start + uzp2 v7.8h, v17.8h, v27.8h + uzp1 v25.8h, v17.8h, v27.8h + uzp2 v0.8h, v16.8h, v23.8h + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + st3 { v28.8b, v29.8b, v30.8b }, [x0], #24 + shrn v21.8b, v25.8h, #0x8 + uzp1 v2.8h, v16.8h, v23.8h + shrn v22.8b, v7.8h, #0x4 + shrn v4.8b, v0.8h, #0x4 + xtn v28.8b, v7.8h + xtn v27.8b, v0.8h + shrn v3.8b, v2.8h, #0x8 + sli v21.8b, v28.8b, #0x4 + xtn v2.8b, v2.8h + sli v3.8b, v27.8b, #0x4 + xtn v20.8b, v25.8h + st3 { v20.8b, v21.8b, v22.8b }, [x0], #24 + st3 { v2.8b, v3.8b, v4.8b }, [x0], #24 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_tobytes_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (!MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S new file mode 100644 index 0000000000..9179e3ff94 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_poly_tomont_aarch64_asm.S @@ -0,0 +1,102 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: poly_tomont_aarch64_asm + Description: Convert polynomial to Montgomery domain + Signature: void mlk_poly_tomont_aarch64_asm(int16_t p[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t p[256] + description: Input/output polynomial + Stack: + bytes: 0 +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_KEYPAIR_API) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_poly_tomont_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tomont_aarch64_asm) +MLK_ASM_FN_SYMBOL(poly_tomont_aarch64_asm) + + .cfi_startproc + mov w2, #0xd01 // =3329 + dup v4.8h, w2 + mov w2, #-0x414 // =-1044 + dup v2.8h, w2 + mov w2, #-0x2824 // =-10276 + dup v3.8h, w2 + mov x1, #0x8 // =8 + ldr q18, [x0, #0x20] + ldr q0, [x0, #0x10] + ldr q16, [x0], #0x40 + sqrdmulh v23.8h, v0.8h, v3.8h + mul v26.8h, v0.8h, v2.8h + sqrdmulh v19.8h, v16.8h, v3.8h + mls v26.8h, v23.8h, v4.h[0] + mul v29.8h, v16.8h, v2.8h + ldur q16, [x0, #-0x10] + mls v29.8h, v19.8h, v4.h[0] + stur q26, [x0, #-0x30] + sqrdmulh v26.8h, v18.8h, v3.8h + mul v18.8h, v18.8h, v2.8h + stur q29, [x0, #-0x40] + sqrdmulh v29.8h, v16.8h, v3.8h + mls v18.8h, v26.8h, v4.h[0] + sub x1, x1, #0x1 + +Lmlk_poly_tomont_loop: + ldr q19, [x0, #0x10] + mul v26.8h, v16.8h, v2.8h + ldr q23, [x0, #0x20] + ldr q17, [x0], #0x40 + mls v26.8h, v29.8h, v4.h[0] + ldur q16, [x0, #-0x10] + sqrdmulh v28.8h, v19.8h, v3.8h + stur q18, [x0, #-0x60] + mul v0.8h, v19.8h, v2.8h + stur q26, [x0, #-0x50] + sqrdmulh v24.8h, v23.8h, v3.8h + mul v18.8h, v23.8h, v2.8h + sqrdmulh v22.8h, v17.8h, v3.8h + mul v26.8h, v17.8h, v2.8h + mls v0.8h, v28.8h, v4.h[0] + mls v26.8h, v22.8h, v4.h[0] + sqrdmulh v29.8h, v16.8h, v3.8h + stur q0, [x0, #-0x30] + mls v18.8h, v24.8h, v4.h[0] + stur q26, [x0, #-0x40] + sub x1, x1, #0x1 + cbnz x1, Lmlk_poly_tomont_loop + mul v16.8h, v16.8h, v2.8h + stur q18, [x0, #-0x20] + mls v16.8h, v29.8h, v4.h[0] + stur q16, [x0, #-0x10] + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_tomont_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S new file mode 100644 index 0000000000..41bd1581ed --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S @@ -0,0 +1,264 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=2 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(int16_t r[256], const int16_t a[512], const int16_t b[512], const int16_t b_cache[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t a[512] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t b[512] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t b_cache[256] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + mov x13, #0x10 // =16 + ldr q12, [x1], #0x20 + ldur q9, [x1, #-0x10] + ldr q22, [x2], #0x20 + ldur q30, [x2, #-0x10] + ldr q6, [x5], #0x20 + ldr q7, [x4, #0x10] + ldr q8, [x4], #0x20 + ldur q23, [x5, #-0x10] + uzp1 v16.8h, v12.8h, v9.8h + uzp2 v14.8h, v12.8h, v9.8h + uzp2 v13.8h, v22.8h, v30.8h + uzp1 v18.8h, v22.8h, v30.8h + ld1 { v27.8h }, [x3], #16 + ld1 { v17.8h }, [x6], #16 + smull2 v4.4s, v16.8h, v18.8h + ldr q31, [x1, #0x10] + smull v19.4s, v16.4h, v13.4h + ldr q24, [x1], #0x20 + smlal v19.4s, v14.4h, v18.4h + ldr q22, [x2], #0x20 + smlal2 v4.4s, v14.8h, v27.8h + uzp2 v5.8h, v6.8h, v23.8h + smull2 v29.4s, v16.8h, v13.8h + uzp2 v26.8h, v8.8h, v7.8h + smlal2 v29.4s, v14.8h, v18.8h + uzp1 v30.8h, v24.8h, v31.8h + uzp1 v8.8h, v8.8h, v7.8h + smull v11.4s, v16.4h, v18.4h + smlal v11.4s, v14.4h, v27.4h + ldur q1, [x2, #-0x10] + uzp1 v28.8h, v6.8h, v23.8h + smlal2 v29.4s, v8.8h, v5.8h + ldr q25, [x5], #0x20 + smlal v19.4s, v8.4h, v5.4h + ldr q3, [x4, #0x10] + smlal2 v29.4s, v26.8h, v28.8h + uzp1 v27.8h, v22.8h, v1.8h + smlal v19.4s, v26.4h, v28.4h + ldr q12, [x4], #0x20 + smlal2 v4.4s, v8.8h, v28.8h + ldur q21, [x5, #-0x10] + smlal2 v4.4s, v26.8h, v17.8h + smlal v11.4s, v8.4h, v28.4h + ld1 { v15.8h }, [x6], #16 + smlal v11.4s, v26.4h, v17.4h + ld1 { v20.8h }, [x3], #16 + uzp1 v28.8h, v19.8h, v29.8h + smull2 v23.4s, v30.8h, v27.8h + smull v26.4s, v30.4h, v27.4h + uzp2 v16.8h, v22.8h, v1.8h + mul v28.8h, v28.8h, v2.8h + uzp1 v10.8h, v11.8h, v4.8h + smull2 v8.4s, v30.8h, v16.8h + mul v13.8h, v10.8h, v2.8h + smlal v19.4s, v28.4h, v0.4h + smlal2 v29.4s, v28.8h, v0.8h + smull v18.4s, v30.4h, v16.4h + uzp1 v30.8h, v25.8h, v21.8h + smlal v11.4s, v13.4h, v0.4h + uzp2 v6.8h, v24.8h, v31.8h + uzp1 v16.8h, v12.8h, v3.8h + smlal2 v4.4s, v13.8h, v0.8h + uzp2 v17.8h, v25.8h, v21.8h + smlal2 v8.4s, v6.8h, v27.8h + uzp2 v12.8h, v12.8h, v3.8h + smlal v18.4s, v6.4h, v27.4h + uzp2 v9.8h, v19.8h, v29.8h + smlal2 v8.4s, v16.8h, v17.8h + smlal2 v8.4s, v12.8h, v30.8h + uzp2 v19.8h, v11.8h, v4.8h + sub x13, x13, #0x2 + +Lmlk_polyvec_basemul_acc_montgomery_cached_k2_loop_start: + smlal v18.4s, v16.4h, v17.4h + ldr q7, [x4], #0x20 + ldr q10, [x2, #0x10] + smlal v18.4s, v12.4h, v30.4h + smlal2 v23.4s, v6.8h, v20.8h + ldr q14, [x2], #0x20 + smlal2 v23.4s, v16.8h, v30.8h + zip1 v25.8h, v19.8h, v9.8h + zip2 v3.8h, v19.8h, v9.8h + smlal2 v23.4s, v12.8h, v15.8h + smlal v26.4s, v6.4h, v20.4h + uzp1 v5.8h, v18.8h, v8.8h + uzp2 v21.8h, v14.8h, v10.8h + smlal v26.4s, v16.4h, v30.4h + str q25, [x0], #0x20 + mul v29.8h, v5.8h, v2.8h + uzp1 v24.8h, v14.8h, v10.8h + stur q3, [x0, #-0x10] + smlal v26.4s, v12.4h, v15.4h + ld1 { v15.8h }, [x6], #16 + ldr q28, [x1, #0x10] + ldr q11, [x1], #0x20 + ldr q13, [x5], #0x20 + ldur q27, [x4, #-0x10] + smlal2 v8.4s, v29.8h, v0.8h + ldur q22, [x5, #-0x10] + smlal v18.4s, v29.4h, v0.4h + uzp1 v4.8h, v26.8h, v23.8h + uzp1 v1.8h, v11.8h, v28.8h + uzp2 v6.8h, v11.8h, v28.8h + uzp1 v16.8h, v7.8h, v27.8h + mul v31.8h, v4.8h, v2.8h + uzp2 v17.8h, v13.8h, v22.8h + ld1 { v20.8h }, [x3], #16 + uzp2 v9.8h, v18.8h, v8.8h + smull2 v8.4s, v1.8h, v21.8h + uzp1 v30.8h, v13.8h, v22.8h + smlal2 v8.4s, v6.8h, v24.8h + smlal2 v8.4s, v16.8h, v17.8h + uzp2 v12.8h, v7.8h, v27.8h + smlal v26.4s, v31.4h, v0.4h + smlal2 v23.4s, v31.8h, v0.8h + smull v18.4s, v1.4h, v21.4h + smlal v18.4s, v6.4h, v24.4h + smlal2 v8.4s, v12.8h, v30.8h + uzp2 v19.8h, v26.8h, v23.8h + smull2 v23.4s, v1.8h, v24.8h + smull v26.4s, v1.4h, v24.4h + subs x13, x13, #0x1 + cbnz x13, Lmlk_polyvec_basemul_acc_montgomery_cached_k2_loop_start + smlal v26.4s, v6.4h, v20.4h + smlal2 v23.4s, v6.8h, v20.8h + smlal v26.4s, v16.4h, v30.4h + smlal2 v23.4s, v16.8h, v30.8h + smlal v26.4s, v12.4h, v15.4h + smlal2 v23.4s, v12.8h, v15.8h + smlal v18.4s, v16.4h, v17.4h + smlal v18.4s, v12.4h, v30.4h + zip1 v12.8h, v19.8h, v9.8h + str q12, [x0], #0x20 + uzp1 v12.8h, v26.8h, v23.8h + mul v6.8h, v12.8h, v2.8h + uzp1 v12.8h, v18.8h, v8.8h + mul v12.8h, v12.8h, v2.8h + smlal v26.4s, v6.4h, v0.4h + smlal2 v23.4s, v6.8h, v0.8h + smlal2 v8.4s, v12.8h, v0.8h + smlal v18.4s, v12.4h, v0.4h + zip2 v12.8h, v19.8h, v9.8h + uzp2 v6.8h, v26.8h, v23.8h + stur q12, [x0, #-0x10] + uzp2 v12.8h, v18.8h, v8.8h + zip2 v1.8h, v6.8h, v12.8h + zip1 v12.8h, v6.8h, v12.8h + str q1, [x0, #0x10] + str q12, [x0], #0x20 + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm) + + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S new file mode 100644 index 0000000000..7dcc8aead2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S @@ -0,0 +1,317 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=3 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(int16_t r[256], const int16_t a[768], const int16_t b[768], const int16_t b_cache[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t a[768] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t b[768] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t b_cache[384] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + add x7, x1, #0x400 + add x8, x2, #0x400 + add x9, x3, #0x200 + mov x13, #0x10 // =16 + ldr q6, [x7], #0x20 + ldr q19, [x2, #0x10] + ldr q23, [x1], #0x20 + ldur q14, [x1, #-0x10] + ldr q17, [x2], #0x20 + ldr q11, [x4, #0x10] + ldur q28, [x7, #-0x10] + ld1 { v30.8h }, [x3], #16 + ldr q26, [x4], #0x20 + ldr q16, [x8, #0x10] + uzp1 v8.8h, v23.8h, v14.8h + ldr q22, [x5, #0x10] + ldr q18, [x5], #0x20 + uzp1 v20.8h, v17.8h, v19.8h + uzp2 v24.8h, v23.8h, v14.8h + ldr q31, [x8], #0x20 + smull2 v4.4s, v8.8h, v20.8h + uzp1 v25.8h, v26.8h, v11.8h + smull v13.4s, v8.4h, v20.4h + ld1 { v23.8h }, [x6], #16 + uzp1 v1.8h, v18.8h, v22.8h + smlal v13.4s, v24.4h, v30.4h + smlal2 v4.4s, v24.8h, v30.8h + uzp2 v5.8h, v26.8h, v11.8h + smlal2 v4.4s, v25.8h, v1.8h + uzp1 v29.8h, v6.8h, v28.8h + smlal2 v4.4s, v5.8h, v23.8h + ld1 { v7.8h }, [x9], #16 + smlal v13.4s, v25.4h, v1.4h + uzp2 v17.8h, v17.8h, v19.8h + uzp1 v27.8h, v31.8h, v16.8h + smlal v13.4s, v5.4h, v23.4h + uzp2 v22.8h, v18.8h, v22.8h + smull v18.4s, v8.4h, v17.4h + uzp2 v28.8h, v6.8h, v28.8h + smlal v13.4s, v29.4h, v27.4h + smlal2 v4.4s, v29.8h, v27.8h + uzp2 v26.8h, v31.8h, v16.8h + smlal2 v4.4s, v28.8h, v7.8h + ldr q3, [x7, #0x10] + smlal v13.4s, v28.4h, v7.4h + ldr q7, [x1], #0x20 + smlal v18.4s, v24.4h, v20.4h + ldr q15, [x2], #0x20 + smlal v18.4s, v25.4h, v22.4h + smull2 v8.4s, v8.8h, v17.8h + ldur q17, [x1, #-0x10] + uzp1 v23.8h, v13.8h, v4.8h + smlal v18.4s, v5.4h, v1.4h + smlal2 v8.4s, v24.8h, v20.8h + ld1 { v16.8h }, [x3], #16 + mul v23.8h, v23.8h, v2.8h + ldr q19, [x5, #0x10] + ldr q14, [x4, #0x10] + ldr q11, [x4], #0x20 + ldur q20, [x2, #-0x10] + smlal2 v8.4s, v25.8h, v22.8h + smlal2 v8.4s, v5.8h, v1.8h + ldr q22, [x5], #0x20 + uzp1 v1.8h, v7.8h, v17.8h + smlal v18.4s, v29.4h, v26.4h + smlal v13.4s, v23.4h, v0.4h + uzp2 v31.8h, v11.8h, v14.8h + uzp1 v21.8h, v15.8h, v20.8h + smlal2 v4.4s, v23.8h, v0.8h + ld1 { v9.8h }, [x6], #16 + smlal v18.4s, v28.4h, v27.4h + smlal2 v8.4s, v29.8h, v26.8h + ldr q25, [x7], #0x20 + smull v26.4s, v1.4h, v21.4h + uzp1 v24.8h, v22.8h, v19.8h + smlal2 v8.4s, v28.8h, v27.8h + uzp2 v28.8h, v7.8h, v17.8h + uzp1 v29.8h, v11.8h, v14.8h + smull2 v23.4s, v1.8h, v21.8h + ldr q27, [x8], #0x20 + smlal2 v23.4s, v28.8h, v16.8h + ldur q11, [x8, #-0x10] + smlal2 v23.4s, v29.8h, v24.8h + uzp2 v7.8h, v13.8h, v4.8h + uzp2 v19.8h, v22.8h, v19.8h + ld1 { v4.8h }, [x9], #16 + smlal2 v23.4s, v31.8h, v9.8h + uzp1 v13.8h, v25.8h, v3.8h + uzp1 v14.8h, v18.8h, v8.8h + smlal v26.4s, v28.4h, v16.4h + uzp2 v17.8h, v27.8h, v11.8h + uzp2 v20.8h, v15.8h, v20.8h + mul v14.8h, v14.8h, v2.8h + sub x13, x13, #0x2 + +Lmlk_polyvec_basemul_acc_montgomery_cached_k3_loop_start: + uzp1 v6.8h, v27.8h, v11.8h + smlal v26.4s, v29.4h, v24.4h + uzp2 v16.8h, v25.8h, v3.8h + smlal v26.4s, v31.4h, v9.4h + ldr q3, [x7, #0x10] + smlal v26.4s, v13.4h, v6.4h + smlal2 v8.4s, v14.8h, v0.8h + ldr q27, [x8], #0x20 + smlal v18.4s, v14.4h, v0.4h + ldr q25, [x7], #0x20 + smlal2 v23.4s, v13.8h, v6.8h + ldr q11, [x1], #0x20 + smlal2 v23.4s, v16.8h, v4.8h + smlal v26.4s, v16.4h, v4.4h + ldur q22, [x1, #-0x10] + uzp2 v30.8h, v18.8h, v8.8h + smull v18.4s, v1.4h, v20.4h + smlal v18.4s, v28.4h, v21.4h + ldr q14, [x2], #0x20 + smlal v18.4s, v29.4h, v19.4h + zip1 v5.8h, v7.8h, v30.8h + uzp1 v4.8h, v26.8h, v23.8h + smull2 v8.4s, v1.8h, v20.8h + zip2 v10.8h, v7.8h, v30.8h + smlal v18.4s, v31.4h, v24.4h + mul v12.8h, v4.8h, v2.8h + ldr q4, [x5, #0x10] + ldr q20, [x4, #0x10] + ldr q1, [x4], #0x20 + ldur q30, [x2, #-0x10] + smlal2 v8.4s, v28.8h, v21.8h + smlal2 v8.4s, v29.8h, v19.8h + ldr q19, [x5], #0x20 + smlal2 v8.4s, v31.8h, v24.8h + ld1 { v15.8h }, [x3], #16 + uzp2 v31.8h, v1.8h, v20.8h + smlal v26.4s, v12.4h, v0.4h + smlal2 v23.4s, v12.8h, v0.8h + uzp1 v21.8h, v14.8h, v30.8h + uzp1 v29.8h, v1.8h, v20.8h + uzp1 v1.8h, v11.8h, v22.8h + smlal2 v8.4s, v13.8h, v17.8h + ld1 { v9.8h }, [x6], #16 + smlal v18.4s, v13.4h, v17.4h + uzp1 v24.8h, v19.8h, v4.8h + uzp2 v7.8h, v26.8h, v23.8h + smull v26.4s, v1.4h, v21.4h + smlal v18.4s, v16.4h, v6.4h + uzp2 v19.8h, v19.8h, v4.8h + smlal2 v8.4s, v16.8h, v6.8h + uzp2 v28.8h, v11.8h, v22.8h + smull2 v23.4s, v1.8h, v21.8h + uzp1 v13.8h, v25.8h, v3.8h + smlal2 v23.4s, v28.8h, v15.8h + ldur q11, [x8, #-0x10] + smlal2 v23.4s, v29.8h, v24.8h + ld1 { v4.8h }, [x9], #16 + smlal2 v23.4s, v31.8h, v9.8h + uzp1 v12.8h, v18.8h, v8.8h + uzp2 v20.8h, v14.8h, v30.8h + smlal v26.4s, v28.4h, v15.4h + str q5, [x0], #0x20 + mul v14.8h, v12.8h, v2.8h + stur q10, [x0, #-0x10] + uzp2 v17.8h, v27.8h, v11.8h + subs x13, x13, #0x1 + cbnz x13, Lmlk_polyvec_basemul_acc_montgomery_cached_k3_loop_start + uzp2 v3.8h, v25.8h, v3.8h + smull2 v16.4s, v1.8h, v20.8h + smull v25.4s, v1.4h, v20.4h + uzp1 v22.8h, v27.8h, v11.8h + smlal2 v16.4s, v28.8h, v21.8h + smlal v25.4s, v28.4h, v21.4h + smlal2 v16.4s, v29.8h, v19.8h + smlal v25.4s, v29.4h, v19.4h + smlal2 v16.4s, v31.8h, v24.8h + smlal v25.4s, v31.4h, v24.4h + smlal v25.4s, v13.4h, v17.4h + smlal2 v16.4s, v13.8h, v17.8h + smlal2 v16.4s, v3.8h, v22.8h + smlal v25.4s, v3.4h, v22.4h + smlal2 v23.4s, v13.8h, v22.8h + smlal v26.4s, v29.4h, v24.4h + smlal v26.4s, v31.4h, v9.4h + smlal v26.4s, v13.4h, v22.4h + uzp1 v10.8h, v25.8h, v16.8h + smlal2 v23.4s, v3.8h, v4.8h + smlal v26.4s, v3.4h, v4.4h + mul v13.8h, v10.8h, v2.8h + smlal v18.4s, v14.4h, v0.4h + smlal2 v8.4s, v14.8h, v0.8h + uzp1 v3.8h, v26.8h, v23.8h + mul v24.8h, v3.8h, v2.8h + uzp2 v17.8h, v18.8h, v8.8h + smlal v25.4s, v13.4h, v0.4h + smlal2 v16.4s, v13.8h, v0.8h + zip1 v21.8h, v7.8h, v17.8h + zip2 v20.8h, v7.8h, v17.8h + smlal2 v23.4s, v24.8h, v0.8h + str q21, [x0], #0x20 + smlal v26.4s, v24.4h, v0.4h + uzp2 v13.8h, v25.8h, v16.8h + stur q20, [x0, #-0x10] + uzp2 v23.8h, v26.8h, v23.8h + zip1 v18.8h, v23.8h, v13.8h + zip2 v13.8h, v23.8h, v13.8h + str q18, [x0], #0x20 + stur q13, [x0, #-0x10] + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S new file mode 100644 index 0000000000..5d1ff00e94 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S @@ -0,0 +1,371 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + */ + +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm + Description: Re-implementation of asymmetric base multiplication following @[NeonNTT] for k=4 + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(int16_t r[256], const int16_t a[1024], const int16_t b[1024], const int16_t b_cache[512]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output polynomial + x1: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t a[1024] + description: Input polynomial vector a + x2: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t b[1024] + description: Input polynomial vector b + x3: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t b_cache[512] + description: Cached values for b + Stack: + bytes: 64 + description: saving callee-saved Neon registers +*/ + +/* Re-implementation of asymmetric base multiplication following @[NeonNTT] */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x40 + .cfi_adjust_cfa_offset 0x40 + stp d8, d9, [sp] + .cfi_rel_offset d8, 0x0 + .cfi_rel_offset d9, 0x8 + stp d10, d11, [sp, #0x10] + .cfi_rel_offset d10, 0x10 + .cfi_rel_offset d11, 0x18 + stp d12, d13, [sp, #0x20] + .cfi_rel_offset d12, 0x20 + .cfi_rel_offset d13, 0x28 + stp d14, d15, [sp, #0x30] + .cfi_rel_offset d14, 0x30 + .cfi_rel_offset d15, 0x38 + mov w14, #0xd01 // =3329 + dup v0.8h, w14 + mov w14, #0xcff // =3327 + dup v2.8h, w14 + add x4, x1, #0x200 + add x5, x2, #0x200 + add x6, x3, #0x100 + add x7, x1, #0x400 + add x8, x2, #0x400 + add x9, x3, #0x200 + add x10, x1, #0x600 + add x11, x2, #0x600 + add x12, x3, #0x300 + mov x13, #0x10 // =16 + ldr q28, [x1], #0x20 + ldur q5, [x1, #-0x10] + ldr q31, [x2], #0x20 + ldur q27, [x2, #-0x10] + ldr q7, [x5], #0x20 + ldr q10, [x4], #0x20 + ldur q18, [x5, #-0x10] + ldur q9, [x4, #-0x10] + uzp1 v11.8h, v28.8h, v5.8h + uzp2 v19.8h, v28.8h, v5.8h + uzp2 v4.8h, v31.8h, v27.8h + uzp1 v1.8h, v31.8h, v27.8h + ldr q29, [x7], #0x20 + ldr q28, [x8, #0x10] + uzp1 v24.8h, v10.8h, v9.8h + uzp1 v17.8h, v7.8h, v18.8h + uzp2 v7.8h, v7.8h, v18.8h + ldr q21, [x8], #0x20 + uzp2 v27.8h, v10.8h, v9.8h + ldur q6, [x7, #-0x10] + smull v18.4s, v11.4h, v4.4h + ld1 { v9.8h }, [x3], #16 + smull2 v8.4s, v11.8h, v4.8h + ldr q16, [x11], #0x20 + smlal2 v8.4s, v19.8h, v1.8h + ldur q14, [x11, #-0x10] + smlal v18.4s, v19.4h, v1.4h + uzp1 v10.8h, v21.8h, v28.8h + smlal v18.4s, v24.4h, v7.4h + ldr q4, [x10], #0x20 + smlal2 v8.4s, v24.8h, v7.8h + ld1 { v12.8h }, [x6], #16 + smull2 v23.4s, v11.8h, v1.8h + uzp2 v13.8h, v29.8h, v6.8h + smull v26.4s, v11.4h, v1.4h + uzp1 v29.8h, v29.8h, v6.8h + smlal v26.4s, v19.4h, v9.4h + ldur q15, [x10, #-0x10] + smlal2 v23.4s, v19.8h, v9.8h + uzp2 v9.8h, v21.8h, v28.8h + smlal v18.4s, v27.4h, v17.4h + uzp2 v6.8h, v16.8h, v14.8h + uzp1 v21.8h, v16.8h, v14.8h + smlal2 v8.4s, v27.8h, v17.8h + smlal2 v8.4s, v29.8h, v9.8h + uzp1 v30.8h, v4.8h, v15.8h + uzp2 v16.8h, v4.8h, v15.8h + smlal v18.4s, v29.4h, v9.4h + smlal2 v8.4s, v13.8h, v10.8h + ld1 { v15.8h }, [x9], #16 + smlal v18.4s, v13.4h, v10.4h + ldr q11, [x4], #0x20 + smlal v18.4s, v30.4h, v6.4h + ldr q7, [x2], #0x20 + smlal2 v8.4s, v30.8h, v6.8h + ld1 { v9.8h }, [x12], #16 + smlal2 v23.4s, v24.8h, v17.8h + ldur q4, [x2, #-0x10] + smlal v26.4s, v24.4h, v17.4h + ldur q25, [x4, #-0x10] + smlal2 v8.4s, v16.8h, v21.8h + ldr q5, [x5], #0x20 + smlal v18.4s, v16.4h, v21.4h + ldur q22, [x5, #-0x10] + smlal v26.4s, v27.4h, v12.4h + ldr q19, [x1, #0x10] + smlal v26.4s, v29.4h, v10.4h + ld1 { v20.8h }, [x3], #16 + smlal v26.4s, v13.4h, v15.4h + uzp1 v24.8h, v7.8h, v4.8h + smlal2 v23.4s, v27.8h, v12.8h + uzp1 v28.8h, v18.8h, v8.8h + smlal v26.4s, v30.4h, v21.4h + uzp2 v27.8h, v11.8h, v25.8h + smlal2 v23.4s, v29.8h, v10.8h + uzp2 v31.8h, v7.8h, v4.8h + smlal2 v23.4s, v13.8h, v15.8h + uzp1 v14.8h, v5.8h, v22.8h + uzp1 v17.8h, v11.8h, v25.8h + smlal v26.4s, v16.4h, v9.4h + mul v29.8h, v28.8h, v2.8h + sub x13, x13, #0x2 + +Lmlk_polyvec_basemul_acc_montgomery_cached_k4_loop_start: + smlal2 v23.4s, v30.8h, v21.8h + ldr q11, [x1], #0x20 + uzp2 v15.8h, v5.8h, v22.8h + smlal v18.4s, v29.4h, v0.4h + ldr q12, [x7], #0x20 + smlal2 v8.4s, v29.8h, v0.8h + ldur q3, [x7, #-0x10] + ldr q21, [x8], #0x20 + uzp1 v29.8h, v11.8h, v19.8h + ldur q13, [x8, #-0x10] + uzp2 v5.8h, v11.8h, v19.8h + smlal2 v23.4s, v16.8h, v9.8h + uzp2 v28.8h, v18.8h, v8.8h + smull2 v8.4s, v29.8h, v31.8h + smlal2 v8.4s, v5.8h, v24.8h + uzp1 v7.8h, v12.8h, v3.8h + smlal2 v8.4s, v17.8h, v15.8h + uzp2 v11.8h, v21.8h, v13.8h + uzp1 v4.8h, v26.8h, v23.8h + smlal2 v8.4s, v27.8h, v14.8h + smlal2 v8.4s, v7.8h, v11.8h + mul v6.8h, v4.8h, v2.8h + ldr q19, [x11], #0x20 + uzp2 v25.8h, v12.8h, v3.8h + ldr q12, [x10], #0x20 + smull v18.4s, v29.4h, v31.4h + ldur q3, [x10, #-0x10] + smlal v18.4s, v5.4h, v24.4h + uzp1 v4.8h, v21.8h, v13.8h + smlal v18.4s, v17.4h, v15.4h + ldur q13, [x11, #-0x10] + ld1 { v1.8h }, [x6], #16 + smlal v26.4s, v6.4h, v0.4h + smlal2 v23.4s, v6.8h, v0.8h + ld1 { v10.8h }, [x9], #16 + smlal v18.4s, v27.4h, v14.4h + uzp1 v30.8h, v12.8h, v3.8h + smlal2 v8.4s, v25.8h, v4.8h + uzp2 v31.8h, v19.8h, v13.8h + smlal v18.4s, v7.4h, v11.4h + ld1 { v9.8h }, [x12], #16 + smlal v18.4s, v25.4h, v4.4h + uzp1 v21.8h, v19.8h, v13.8h + uzp2 v16.8h, v12.8h, v3.8h + smlal v18.4s, v30.4h, v31.4h + smlal2 v8.4s, v30.8h, v31.8h + uzp2 v31.8h, v26.8h, v23.8h + smlal2 v8.4s, v16.8h, v21.8h + smlal v18.4s, v16.4h, v21.4h + zip1 v15.8h, v31.8h, v28.8h + ldr q19, [x1, #0x10] + smull2 v23.4s, v29.8h, v24.8h + smull v26.4s, v29.4h, v24.4h + ldr q3, [x2, #0x10] + smlal v26.4s, v5.4h, v20.4h + ldr q11, [x2], #0x20 + uzp1 v6.8h, v18.8h, v8.8h + smlal v26.4s, v17.4h, v14.4h + smlal v26.4s, v27.4h, v1.4h + zip2 v13.8h, v31.8h, v28.8h + smlal v26.4s, v7.4h, v4.4h + str q15, [x0], #0x20 + smlal v26.4s, v25.4h, v10.4h + stur q13, [x0, #-0x10] + mul v29.8h, v6.8h, v2.8h + uzp1 v24.8h, v11.8h, v3.8h + uzp2 v31.8h, v11.8h, v3.8h + ldr q11, [x4], #0x20 + smlal2 v23.4s, v5.8h, v20.8h + ldur q28, [x4, #-0x10] + smlal2 v23.4s, v17.8h, v14.8h + ldr q5, [x5], #0x20 + smlal2 v23.4s, v27.8h, v1.8h + ldur q22, [x5, #-0x10] + smlal v26.4s, v30.4h, v21.4h + ld1 { v20.8h }, [x3], #16 + smlal v26.4s, v16.4h, v9.4h + uzp1 v17.8h, v11.8h, v28.8h + smlal2 v23.4s, v7.8h, v4.8h + uzp2 v27.8h, v11.8h, v28.8h + smlal2 v23.4s, v25.8h, v10.8h + uzp1 v14.8h, v5.8h, v22.8h + subs x13, x13, #0x1 + cbnz x13, Lmlk_polyvec_basemul_acc_montgomery_cached_k4_loop_start + smlal v18.4s, v29.4h, v0.4h + ldr q11, [x1], #0x20 + uzp2 v28.8h, v5.8h, v22.8h + smlal2 v23.4s, v30.8h, v21.8h + smlal2 v8.4s, v29.8h, v0.8h + ldr q15, [x8, #0x10] + smlal2 v23.4s, v16.8h, v9.8h + ldr q21, [x8], #0x20 + uzp1 v22.8h, v11.8h, v19.8h + uzp2 v12.8h, v11.8h, v19.8h + ldr q1, [x7, #0x10] + ld1 { v6.8h }, [x6], #16 + uzp2 v3.8h, v18.8h, v8.8h + smull v9.4s, v22.4h, v31.4h + smull2 v18.4s, v22.8h, v31.8h + ldr q16, [x7], #0x20 + smull v19.4s, v22.4h, v24.4h + uzp1 v30.8h, v21.8h, v15.8h + uzp2 v25.8h, v21.8h, v15.8h + smull2 v8.4s, v22.8h, v24.8h + smlal v19.4s, v12.4h, v20.4h + ldr q13, [x10, #0x10] + smlal2 v8.4s, v12.8h, v20.8h + uzp1 v29.8h, v16.8h, v1.8h + smlal2 v18.4s, v12.8h, v24.8h + ldr q5, [x10], #0x20 + smlal v9.4s, v12.4h, v24.4h + ldr q4, [x11], #0x20 + smlal v9.4s, v17.4h, v28.4h + ldur q22, [x11, #-0x10] + smlal2 v18.4s, v17.8h, v28.8h + uzp2 v16.8h, v16.8h, v1.8h + smlal v19.4s, v17.4h, v14.4h + ld1 { v28.8h }, [x9], #16 + smlal2 v8.4s, v17.8h, v14.8h + uzp1 v7.8h, v5.8h, v13.8h + smlal v9.4s, v27.4h, v14.4h + uzp1 v17.8h, v4.8h, v22.8h + smlal2 v18.4s, v27.8h, v14.8h + uzp2 v12.8h, v5.8h, v13.8h + uzp2 v21.8h, v4.8h, v22.8h + smlal v19.4s, v27.4h, v6.4h + smlal2 v8.4s, v27.8h, v6.8h + ld1 { v15.8h }, [x12], #16 + smlal v19.4s, v29.4h, v30.4h + uzp1 v20.8h, v26.8h, v23.8h + smlal v9.4s, v29.4h, v25.4h + smlal2 v18.4s, v29.8h, v25.8h + smlal2 v8.4s, v29.8h, v30.8h + smlal v19.4s, v16.4h, v28.4h + smlal2 v8.4s, v16.8h, v28.8h + smlal2 v18.4s, v16.8h, v30.8h + smlal v9.4s, v16.4h, v30.4h + smlal v9.4s, v7.4h, v21.4h + smlal2 v18.4s, v7.8h, v21.8h + smlal2 v8.4s, v7.8h, v17.8h + smlal v19.4s, v7.4h, v17.4h + smlal v19.4s, v12.4h, v15.4h + smlal2 v8.4s, v12.8h, v15.8h + smlal2 v18.4s, v12.8h, v17.8h + smlal v9.4s, v12.4h, v17.4h + mul v6.8h, v20.8h, v2.8h + uzp1 v4.8h, v19.8h, v8.8h + mul v17.8h, v4.8h, v2.8h + uzp1 v12.8h, v9.8h, v18.8h + smlal v26.4s, v6.4h, v0.4h + mul v21.8h, v12.8h, v2.8h + smlal2 v23.4s, v6.8h, v0.8h + smlal2 v8.4s, v17.8h, v0.8h + smlal v19.4s, v17.4h, v0.4h + smlal2 v18.4s, v21.8h, v0.8h + uzp2 v23.8h, v26.8h, v23.8h + smlal v9.4s, v21.4h, v0.4h + zip2 v12.8h, v23.8h, v3.8h + zip1 v22.8h, v23.8h, v3.8h + uzp2 v14.8h, v19.8h, v8.8h + uzp2 v18.8h, v9.8h, v18.8h + str q12, [x0, #0x10] + str q22, [x0], #0x20 + zip2 v24.8h, v14.8h, v18.8h + zip1 v21.8h, v14.8h, v18.8h + str q24, [x0, #0x10] + str q21, [x0], #0x20 + ldp d8, d9, [sp] + .cfi_restore d8 + .cfi_restore d9 + ldp d10, d11, [sp, #0x10] + .cfi_restore d10 + .cfi_restore d11 + ldp d12, d13, [sp, #0x20] + .cfi_restore d12 + .cfi_restore d13 + ldp d14, d15, [sp, #0x30] + .cfi_restore d14 + .cfi_restore d15 + add sp, sp, #0x40 + .cfi_adjust_cfa_offset -0x40 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S new file mode 100644 index 0000000000..2cd1ca9fec --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/mlkem_rej_uniform_aarch64_asm.S @@ -0,0 +1,226 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/*yaml + Name: rej_uniform_aarch64_asm + Description: Run rejection sampling on uniform random bytes to generate uniform random integers mod q + Signature: uint64_t mlk_rej_uniform_aarch64_asm(int16_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[4096]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + Features: [NEON] + x0: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t r[256] + description: Output buffer + x1: + type: buffer + size_bytes: x2 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + x2: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be multiple of 24) + test_with: 504 # MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE + x3: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t table[4096] + description: Lookup table + Stack: + bytes: 576 + description: register preservation and temporary storage +*/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/aarch64_opt/src/mlkem_rej_uniform_aarch64_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(rej_uniform_aarch64_asm) +MLK_ASM_FN_SYMBOL(rej_uniform_aarch64_asm) + + .cfi_startproc + sub sp, sp, #0x240 + .cfi_adjust_cfa_offset 0x240 + mov x7, #0x1 // =1 + movk x7, #0x2, lsl #16 + movk x7, #0x4, lsl #32 + movk x7, #0x8, lsl #48 + mov v31.d[0], x7 + mov x7, #0x10 // =16 + movk x7, #0x20, lsl #16 + movk x7, #0x40, lsl #32 + movk x7, #0x80, lsl #48 + mov v31.d[1], x7 + mov w11, #0xd01 // =3329 + dup v30.8h, w11 + mov x8, sp + mov x7, x8 + mov x11, #0x0 // =0 + eor v16.16b, v16.16b, v16.16b + +Lmlk_rej_uniform_initial_zero: + str q16, [x7], #0x40 + stur q16, [x7, #-0x30] + stur q16, [x7, #-0x20] + stur q16, [x7, #-0x10] + add x11, x11, #0x20 + cmp x11, #0x100 + b.lt Lmlk_rej_uniform_initial_zero + mov x7, x8 + mov x9, #0x0 // =0 + mov x4, #0x100 // =256 + cmp x2, #0x30 + b.lo Lmlk_rej_uniform_loop48_end + +Lmlk_rej_uniform_loop48: + cmp x9, x4 + b.hs Lmlk_rej_uniform_memory_copy + sub x2, x2, #0x30 + ld3 { v0.16b, v1.16b, v2.16b }, [x1], #48 + zip1 v4.16b, v0.16b, v1.16b + zip2 v5.16b, v0.16b, v1.16b + zip1 v6.16b, v1.16b, v2.16b + zip2 v7.16b, v1.16b, v2.16b + bic v4.8h, #0xf0, lsl #8 + bic v5.8h, #0xf0, lsl #8 + ushr v6.8h, v6.8h, #0x4 + ushr v7.8h, v7.8h, #0x4 + zip1 v16.8h, v4.8h, v6.8h + zip2 v17.8h, v4.8h, v6.8h + zip1 v18.8h, v5.8h, v7.8h + zip2 v19.8h, v5.8h, v7.8h + cmhi v4.8h, v30.8h, v16.8h + cmhi v5.8h, v30.8h, v17.8h + cmhi v6.8h, v30.8h, v18.8h + cmhi v7.8h, v30.8h, v19.8h + and v4.16b, v4.16b, v31.16b + and v5.16b, v5.16b, v31.16b + and v6.16b, v6.16b, v31.16b + and v7.16b, v7.16b, v31.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + uaddlv s22, v6.8h + uaddlv s23, v7.8h + fmov w12, s20 + fmov w13, s21 + fmov w14, s22 + fmov w15, s23 + ldr q24, [x3, x12, lsl #4] + ldr q25, [x3, x13, lsl #4] + ldr q26, [x3, x14, lsl #4] + ldr q27, [x3, x15, lsl #4] + cnt v4.16b, v4.16b + cnt v5.16b, v5.16b + cnt v6.16b, v6.16b + cnt v7.16b, v7.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + uaddlv s22, v6.8h + uaddlv s23, v7.8h + fmov w12, s20 + fmov w13, s21 + fmov w14, s22 + fmov w15, s23 + tbl v16.16b, { v16.16b }, v24.16b + tbl v17.16b, { v17.16b }, v25.16b + tbl v18.16b, { v18.16b }, v26.16b + tbl v19.16b, { v19.16b }, v27.16b + st1 { v16.8h }, [x7] + add x7, x7, x12, lsl #1 + st1 { v17.8h }, [x7] + add x7, x7, x13, lsl #1 + st1 { v18.8h }, [x7] + add x7, x7, x14, lsl #1 + st1 { v19.8h }, [x7] + add x7, x7, x15, lsl #1 + add x12, x12, x13 + add x14, x14, x15 + add x9, x9, x12 + add x9, x9, x14 + cmp x2, #0x30 + b.hs Lmlk_rej_uniform_loop48 + +Lmlk_rej_uniform_loop48_end: + cmp x9, x4 + b.hs Lmlk_rej_uniform_memory_copy + cmp x2, #0x18 + b.lo Lmlk_rej_uniform_memory_copy + ld3 { v0.8b, v1.8b, v2.8b }, [x1], #24 + zip1 v4.16b, v0.16b, v1.16b + zip1 v5.16b, v1.16b, v2.16b + bic v4.8h, #0xf0, lsl #8 + ushr v5.8h, v5.8h, #0x4 + zip1 v16.8h, v4.8h, v5.8h + zip2 v17.8h, v4.8h, v5.8h + cmhi v4.8h, v30.8h, v16.8h + cmhi v5.8h, v30.8h, v17.8h + and v4.16b, v4.16b, v31.16b + and v5.16b, v5.16b, v31.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + fmov w12, s20 + fmov w13, s21 + ldr q24, [x3, x12, lsl #4] + ldr q25, [x3, x13, lsl #4] + cnt v4.16b, v4.16b + cnt v5.16b, v5.16b + uaddlv s20, v4.8h + uaddlv s21, v5.8h + fmov w12, s20 + fmov w13, s21 + tbl v16.16b, { v16.16b }, v24.16b + tbl v17.16b, { v17.16b }, v25.16b + st1 { v16.8h }, [x7] + add x7, x7, x12, lsl #1 + st1 { v17.8h }, [x7] + add x9, x9, x12 + add x9, x9, x13 + +Lmlk_rej_uniform_memory_copy: + cmp x9, x4 + csel x9, x9, x4, lo + mov x11, #0x0 // =0 + mov x7, x8 + +Lmlk_rej_uniform_final_copy: + ldr q16, [x7], #0x40 + ldur q17, [x7, #-0x30] + ldur q18, [x7, #-0x20] + ldur q19, [x7, #-0x10] + str q16, [x0], #0x40 + stur q17, [x0, #-0x30] + stur q18, [x0, #-0x20] + stur q19, [x0, #-0x10] + add x11, x11, #0x20 + cmp x11, #0x100 + b.lt Lmlk_rej_uniform_final_copy + mov x0, x9 + +Lmlk_rej_uniform_return: + add sp, sp, #0x240 + .cfi_adjust_cfa_offset -0x240 + ret + .cfi_endproc + +MLK_ASM_FN_SIZE(rej_uniform_aarch64_asm) + +#endif /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c b/src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c new file mode 100644 index 0000000000..e1f48dc77d --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/aarch64/src/rej_uniform_table.c @@ -0,0 +1,543 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_AARCH64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_aarch64.h" + +/* + * Lookup table used by rejection sampling of the public matrix. + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_rej_uniform_table[4096] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 0 */, + 0, 1, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 1 */, + 2, 3, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 2 */, + 0, 1, 2, 3, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 3 */, + 4, 5, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 4 */, + 0, 1, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 5 */, + 2, 3, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 6 */, + 0, 1, 2, 3, 4, 5, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 7 */, + 6, 7, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 8 */, + 0, 1, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 9 */, + 2, 3, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 10 */, + 0, 1, 2, 3, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 11 */, + 4, 5, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 12 */, + 0, 1, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 13 */, + 2, 3, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 14 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 255, 255, 255, 255, 255, 255, 255, 255 /* 15 */, + 8, 9, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 16 */, + 0, 1, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 17 */, + 2, 3, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 18 */, + 0, 1, 2, 3, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 19 */, + 4, 5, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 20 */, + 0, 1, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 21 */, + 2, 3, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 22 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 23 */, + 6, 7, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 24 */, + 0, 1, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 25 */, + 2, 3, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 26 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 27 */, + 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 28 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 29 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 30 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 255, 255, 255, 255, 255, 255 /* 31 */, + 10, 11, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 32 */, + 0, 1, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 33 */, + 2, 3, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 34 */, + 0, 1, 2, 3, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 35 */, + 4, 5, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 36 */, + 0, 1, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 37 */, + 2, 3, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 38 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 39 */, + 6, 7, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 40 */, + 0, 1, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 41 */, + 2, 3, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 42 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 43 */, + 4, 5, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 44 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 45 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 46 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 255, 255, 255, 255, 255, 255 /* 47 */, + 8, 9, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 48 */, + 0, 1, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 49 */, + 2, 3, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 50 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 51 */, + 4, 5, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 52 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 53 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 54 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 55 */, + 6, 7, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 56 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 57 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 58 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 59 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 60 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 61 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 62 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 255, 255, 255, 255 /* 63 */, + 12, 13, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 64 */, + 0, 1, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 65 */, + 2, 3, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 66 */, + 0, 1, 2, 3, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 67 */, + 4, 5, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 68 */, + 0, 1, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 69 */, + 2, 3, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 70 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 71 */, + 6, 7, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 72 */, + 0, 1, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 73 */, + 2, 3, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 74 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 75 */, + 4, 5, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 76 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 77 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 78 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 255, 255, 255, 255, 255, 255 /* 79 */, + 8, 9, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 80 */, + 0, 1, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 81 */, + 2, 3, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 82 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 83 */, + 4, 5, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 84 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 85 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 86 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 87 */, + 6, 7, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 88 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 89 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 90 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 91 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 92 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 93 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 94 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 255, 255, 255, 255 /* 95 */, + 10, 11, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 96 */, + 0, 1, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 97 */, + 2, 3, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 98 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 99 */, + 4, 5, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 100 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 101 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 102 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 103 */, + 6, 7, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 104 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 105 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 106 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 107 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 108 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 109 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 110 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 255, 255, 255, 255 /* 111 */, + 8, 9, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 112 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 113 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 114 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 115 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 116 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 117 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 118 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 119 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 120 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 121 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 122 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 123 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 124 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 125 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 126 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 255, 255 /* 127 */, + 14, 15, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 128 */, + 0, 1, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 129 */, + 2, 3, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 130 */, + 0, 1, 2, 3, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 131 */, + 4, 5, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 132 */, + 0, 1, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 133 */, + 2, 3, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 134 */, + 0, 1, 2, 3, 4, 5, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 135 */, + 6, 7, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 136 */, + 0, 1, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 137 */, + 2, 3, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 138 */, + 0, 1, 2, 3, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 139 */, + 4, 5, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 140 */, + 0, 1, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 141 */, + 2, 3, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 142 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 14, 15, 255, 255, 255, 255, 255, 255 /* 143 */, + 8, 9, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 144 */, + 0, 1, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 145 */, + 2, 3, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 146 */, + 0, 1, 2, 3, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 147 */, + 4, 5, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 148 */, + 0, 1, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 149 */, + 2, 3, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 150 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 151 */, + 6, 7, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 152 */, + 0, 1, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 153 */, + 2, 3, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 154 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 155 */, + 4, 5, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 156 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 157 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 158 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 14, 15, 255, 255, 255, 255 /* 159 */, + 10, 11, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 160 */, + 0, 1, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 161 */, + 2, 3, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 162 */, + 0, 1, 2, 3, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 163 */, + 4, 5, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 164 */, + 0, 1, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 165 */, + 2, 3, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 166 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 167 */, + 6, 7, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 168 */, + 0, 1, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 169 */, + 2, 3, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 170 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 171 */, + 4, 5, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 172 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 173 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 174 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 14, 15, 255, 255, 255, 255 /* 175 */, + 8, 9, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 176 */, + 0, 1, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 177 */, + 2, 3, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 178 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 179 */, + 4, 5, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 180 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 181 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 182 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 183 */, + 6, 7, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 184 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 185 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 186 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 187 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 188 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 189 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 190 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 14, 15, 255, 255 /* 191 */, + 12, 13, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 192 */, + 0, 1, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 193 */, + 2, 3, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 194 */, + 0, 1, 2, 3, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 195 */, + 4, 5, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 196 */, + 0, 1, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 197 */, + 2, 3, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 198 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 199 */, + 6, 7, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 200 */, + 0, 1, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 201 */, + 2, 3, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 202 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 203 */, + 4, 5, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 204 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 205 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 206 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 14, 15, 255, 255, 255, 255 /* 207 */, + 8, 9, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 208 */, + 0, 1, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 209 */, + 2, 3, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 210 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 211 */, + 4, 5, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 212 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 213 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 214 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 215 */, + 6, 7, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 216 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 217 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 218 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 219 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 220 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 221 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 222 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 14, 15, 255, 255 /* 223 */, + 10, 11, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 224 */, + 0, 1, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 225 */, + 2, 3, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 226 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 227 */, + 4, 5, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 228 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 229 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 230 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 231 */, + 6, 7, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 232 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 233 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 234 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 235 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 236 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 237 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 238 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 14, 15, 255, 255 /* 239 */, + 8, 9, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 240 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 241 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 242 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 243 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 244 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 245 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 246 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 247 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 248 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 249 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 250 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 251 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 252 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 253 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 254 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 /* 255 */, +}; + +#else /* MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(aarch64_rej_uniform_table) + +#endif /* !(MLK_ARITH_BACKEND_AARCH64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/native/api.h b/src/mlkem-native/mlkem/src/native/api.h new file mode 100644 index 0000000000..f8578763b6 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/api.h @@ -0,0 +1,651 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_NATIVE_API_H +#define MLK_NATIVE_API_H +/* + * Native arithmetic interface + * + * This header is primarily for documentation purposes. + * It should not be included by backend implementations. + * + * To ensure consistency with backends, the header will be + * included automatically after inclusion of the active + * backend, to ensure consistency of function signatures, + * and run sanity checks. + */ + +#include "../cbmc.h" +#include "../common.h" + +/* Backends must return MLK_NATIVE_FUNC_SUCCESS upon success. */ +#define MLK_NATIVE_FUNC_SUCCESS (0) +/* Backends may return MLK_NATIVE_FUNC_FALLBACK to signal to the frontend that + * the target/parameters are unsupported; typically, this would be because of + * dependencies on CPU features not detected on the host CPU. In this case, + * the frontend falls back to the default C implementation. */ +#define MLK_NATIVE_FUNC_FALLBACK (-1) + + +/* Absolute exclusive upper bound for the output of the inverse NTT + * + * NOTE: This is the same bound as in poly.h and has to be kept + * in sync. */ +#define MLK_INVNTT_BOUND (8 * MLKEM_Q) + +/* Absolute exclusive upper bound for the output of the forward NTT + * + * NOTE: This is the same bound as in poly.h and has to be kept + * in sync. */ +#define MLK_NTT_BOUND (8 * MLKEM_Q) + +/* + * This is the C<->native interface allowing for the drop-in of + * native code for performance critical arithmetic components of ML-KEM. + * + * A _backend_ is a specific implementation of (part of) this interface. + * + * To add a function to a backend, define MLK_USE_NATIVE_XXX and + * implement `static inline xxx(...)` in the profile header. + * + * The only exception is MLK_USE_NATIVE_NTT_CUSTOM_ORDER. This option can + * be set if there are native implementations for all of NTT, invNTT, and + * base multiplication, and allows the native implementation to use a + * custom order of polynomial coefficients in NTT domain -- the use of such + * custom order is not an implementation-detail since the public matrix + * is generated in NTT domain. In this case, a permutation function + * mlk_poly_permute_bitrev_to_custom() needs to be provided that permutes + * polynomials in NTT domain from bitreversed to the custom order. + */ + +/* + * Those functions are meant to be trivial wrappers around the chosen native + * implementation. The are static inline to avoid unnecessary calls. + * The macro before each declaration controls whether a native + * implementation is present. + */ + +#if defined(MLK_USE_NATIVE_NTT) +/** + * Compute the negacyclic number-theoretic transform (NTT) of a polynomial + * in place. + * + * The input polynomial is assumed to be in normal order. The output + * polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_NTT_BOUND)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_NTT */ + +#if defined(MLK_USE_NATIVE_NTT_CUSTOM_ORDER) +/* + * This must only be set if NTT, invNTT, basemul, mulcache, and + * to/from byte stream conversions all have native implementations + * that are adapted to the custom order. + */ +#if !defined(MLK_USE_NATIVE_NTT) || !defined(MLK_USE_NATIVE_INTT) || \ + !defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) || \ + !defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) || \ + !defined(MLK_USE_NATIVE_POLY_TOBYTES) || \ + !defined(MLK_USE_NATIVE_POLY_FROMBYTES) +#error \ + "Invalid native profile: MLK_USE_NATIVE_NTT_CUSTOM_ORDER can only be \ +set if there are native implementations for NTT, invNTT, mulcache, basemul, \ +and to/from bytes conversions." +#endif /* !MLK_USE_NATIVE_NTT || !MLK_USE_NATIVE_INTT || \ + !MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE || \ + !MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED || \ + !MLK_USE_NATIVE_POLY_TOBYTES || !MLK_USE_NATIVE_POLY_FROMBYTES */ + +/** + * When MLK_USE_NATIVE_NTT_CUSTOM_ORDER is defined, convert a polynomial in + * NTT domain from bitreversed order to the custom order output by the + * native NTT. + * + * This must only be defined if there is native code for all of (a) NTT, + * (b) invNTT, (c) basemul, (d) mulcache. + * + * @param[in,out] p Input/output polynomial. + */ +static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t p[MLKEM_N]) +__contract__( + /* We don't specify that this should be a permutation, but only + * that it does not change the bound established at the end of mlk_gen_matrix. */ + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(p, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_NTT_CUSTOM_ORDER */ + +#if defined(MLK_USE_NATIVE_INTT) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place. + * + * The input polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the documentation of + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER for more information. The output + * polynomial is assumed to be in normal order. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLK_INVNTT_BOUND)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_INTT && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(MLK_USE_NATIVE_POLY_REDUCE) +/** + * Apply modular reduction to all coefficients of a polynomial, mapping them + * to unsigned canonical representatives in [0,..,MLKEM_Q-1]. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(p, 0, MLKEM_N, 0, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_POLY_REDUCE */ + +#if defined(MLK_USE_NATIVE_POLY_TOMONT) && !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * @param[in,out] p Input/output polynomial. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t p[MLKEM_N]) +__contract__( + requires(memory_no_alias(p, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(p, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(p, 0, MLKEM_N, MLKEM_Q)) + ensures((return_value == MLK_NATIVE_FUNC_FALLBACK) ==> array_unchanged(p, MLKEM_N)) +); +#endif /* MLK_USE_NATIVE_POLY_TOMONT && !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) +/** + * Compute multiplication cache for a polynomial in NTT domain. + * + * The purpose of the multiplication cache is to cache repeated computations + * required during a base multiplication of polynomials in NTT domain. The + * structure of the multiplication-cache is implementation defined. + * + * @param[out] cache Multiplication cache. + * @param[in] mlk_poly Input polynomial. Must be in NTT domain and in + * bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native( + int16_t cache[MLKEM_N / 2], const int16_t mlk_poly[MLKEM_N]) +__contract__( + requires(memory_no_alias(cache, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(mlk_poly, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(cache, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_abs_bound(cache, 0, MLKEM_N/2, MLKEM_Q)) +); +#endif /* MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE */ + +#if defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +/** + * Compute scalar product of length-2 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 2 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +/** + * Compute scalar product of length-3 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 3 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +/** + * Compute scalar product of length-4 polynomial vectors in NTT domain. + * + * @param[out] r Result of the scalar product. Again in NTT domain, of + * the same ordering as @p a and @p b. + * @param[in] a First polynomial vector operand. Must be in NTT + * domain and in bitreversed order, or of a custom order + * if MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set. See the + * documentation of MLK_USE_NATIVE_NTT_CUSTOM_ORDER for + * more information. + * @param[in] b Second polynomial vector operand. As for @p a. + * @param[in] b_cache Multiplication-cache for @p b. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_bound(a, 0, 4 * MLKEM_N, 0, MLKEM_UINT12_LIMIT)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_FALLBACK || return_value == MLK_NATIVE_FUNC_SUCCESS) +); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ +#endif /* MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED */ + +#if defined(MLK_USE_NATIVE_POLY_TOBYTES) && \ + (!defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + !defined(MLK_CONFIG_NO_ENCAPS_API)) +/** + * Serialization of a polynomial with unsigned canonical coefficients. + * + * @spec{Implements ByteEncode_12 from @[FIPS203, Algorithm 5].} + * + * @param[out] r Output byte array (of MLKEM_POLYBYTES bytes). + * @param[in] a Input polynomial, with each coefficient in [0,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) +); +#endif /* MLK_USE_NATIVE_POLY_TOBYTES && (!MLK_CONFIG_NO_KEYPAIR_API || \ + !MLK_CONFIG_NO_ENCAPS_API) */ + +#if defined(MLK_USE_NATIVE_POLY_FROMBYTES) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/** + * Deserialization of a polynomial. + * + * @spec{Implements ByteDecode_12 from @[FIPS203, Algorithm 6].} + * + * @param[out] a Output polynomial in NTT domain. + * @param[in] r Input byte array (of MLKEM_POLYBYTES bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_frombytes_native( + int16_t a[MLKEM_N], const uint8_t r[MLKEM_POLYBYTES]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(a, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(a, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); +#endif /* MLK_USE_NATIVE_POLY_FROMBYTES && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(MLK_USE_NATIVE_REJ_UNIFORM) +/** + * Run rejection sampling on uniformly random bytes to generate uniformly random + * integers mod MLKEM_Q, represented in [0,..,MLKEM_Q-1]. + * + * @param[out] r Output buffer. + * @param len Requested number of 16-bit integers (uniform mod MLKEM_Q). + * @param[in] buf Input buffer (assumed to be uniform random bytes). + * @param buflen Length of input buffer in bytes. + * + * @retval MLK_NATIVE_FUNC_FALLBACK Native implementation does not support + * the input lengths. + * @retval other Non-negative number of sampled 16-bit + * integers (at most @p len). + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +__contract__( + requires(len <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * len)) + requires(memory_no_alias(buf, buflen)) + assigns(memory_slice(r, sizeof(int16_t) * len)) + ensures(return_value != MLK_NATIVE_FUNC_FALLBACK + ==> (0 <= return_value && return_value <= len)) + ensures(return_value != MLK_NATIVE_FUNC_FALLBACK + ==> array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); +#endif /* MLK_USE_NATIVE_REJ_UNIFORM */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || (MLKEM_K == 2 || MLKEM_K == 3) +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D4) +/** + * Compression (4 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_4 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d4_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D4 */ + +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D10) +/** + * Compression (10 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_10 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d10_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D10 */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D4) +/** + * De-serialization and subsequent decompression (4 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d4. + * + * @spec{Decompress_4 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D4 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d4_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D4 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D10) +/** + * De-serialization and subsequent decompression (10 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d10. + * + * @spec{Decompress_10 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D10 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d10_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D10 */ +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D5) +/** + * Compression (5 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_5 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d5_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D5 */ + +#if defined(MLK_USE_NATIVE_POLY_COMPRESS_D11) +/** + * Compression (11 bits) and subsequent serialization of a polynomial. + * + * @spec{Compress_11 from @[FIPS203, Eq (4.7)].} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d11_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N]) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK)); +#endif /* MLK_USE_NATIVE_POLY_COMPRESS_D11 */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D5) +/** + * De-serialization and subsequent decompression (5 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d5. + * + * @spec{Decompress_5 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D5 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d5_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D5 */ + +#if defined(MLK_USE_NATIVE_POLY_DECOMPRESS_D11) +/** + * De-serialization and subsequent decompression (11 bits) of a polynomial; + * approximate inverse of mlk_poly_compress_d11. + * + * @spec{Decompress_11 from @[FIPS203, Eq (4.8)].} + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_D11 + * bytes). + * + * @retval MLK_NATIVE_FUNC_SUCCESS Operation succeeded. + * @retval MLK_NATIVE_FUNC_FALLBACK Backend declined; caller should fall back. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d11_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value == MLK_NATIVE_FUNC_SUCCESS || return_value == MLK_NATIVE_FUNC_FALLBACK) + ensures((return_value == MLK_NATIVE_FUNC_SUCCESS) ==> array_bound(r, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* MLK_USE_NATIVE_POLY_DECOMPRESS_D11 */ +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_NATIVE_API_H */ diff --git a/src/mlkem-native/mlkem/src/native/meta.h b/src/mlkem-native/mlkem/src/native/meta.h new file mode 100644 index 0000000000..dbd07a5bda --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/meta.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_META_H +#define MLK_NATIVE_META_H + +/* + * Default arithmetic backend + */ +#include "../sys.h" + +#ifdef MLK_SYS_AARCH64_NEON +#include "aarch64/meta.h" +#endif + +/* The x86_64 backend requires toolchain support for the SysV ABI */ +#if defined(MLK_SYS_X86_64_AVX2) && defined(MLK_SYSV_ABI_SUPPORTED) +#include "x86_64/meta.h" +#endif + +#if defined(MLK_SYS_RISCV64_RVV) +#include "riscv64/meta.h" +#endif + +#ifdef MLK_SYS_PPC64LE +#include "ppc64le/meta.h" +#endif + +#endif /* !MLK_NATIVE_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/README.md b/src/mlkem-native/mlkem/src/native/ppc64le/README.md new file mode 100644 index 0000000000..733e32e113 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/README.md @@ -0,0 +1,6 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# ppc64le backend (little endian) + +This directory contains a native backend for little endian POWER 9 (ppc64le) and above systems. +Or, Power systems supporting ISA 2.07 and above. diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/meta.h b/src/mlkem-native/mlkem/src/native/ppc64le/meta.h new file mode 100644 index 0000000000..2f292432df --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/meta.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_PPC64LE_META_H +#define MLK_NATIVE_PPC64LE_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_PPC64LE_DEFAULT + +#define MLK_ARITH_BACKEND_NAME PPC64LE_DEFAULT + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT + +#if !defined(__ASSEMBLER__) +#include +#include "../../common.h" +#include "../../params.h" +#include "../api.h" +#include "src/arith_native_ppc64le.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_ntt_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_intt_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_reduce_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ +#if defined(__POWER8_VECTOR__) + mlk_poly_tomont_ppc_asm(data, mlk_ppc_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +#else + (void)data; + return MLK_NATIVE_FUNC_FALLBACK; +#endif +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_PPC64LE_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h b/src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h new file mode 100644 index 0000000000..3bd47ebd76 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/arith_native_ppc64le.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H +#define MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H + +#include +#include "../../../common.h" +#include "consts.h" + +#define mlk_ntt_ppc_asm MLK_NAMESPACE(ntt_ppc_asm) +void mlk_ntt_ppc_asm(int16_t *, const int16_t *); + +#define mlk_intt_ppc_asm MLK_NAMESPACE(intt_ppc_asm) +void mlk_intt_ppc_asm(int16_t *, const int16_t *); + +#define mlk_reduce_ppc_asm MLK_NAMESPACE(reduce_ppc_asm) +void mlk_reduce_ppc_asm(int16_t *r, const int16_t *); + +#define mlk_poly_tomont_ppc_asm MLK_NAMESPACE(poly_tomont_ppc_asm) +void mlk_poly_tomont_ppc_asm(int16_t *, const int16_t *); + +#endif /* !MLK_NATIVE_PPC64LE_SRC_ARITH_NATIVE_PPC64LE_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c new file mode 100644 index 0000000000..f6e1abd5b9 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.c @@ -0,0 +1,299 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) + +#include "consts.h" + +/* + * Constant table for the PPC64LE NTT/invNTT. + * For the layout, see the offset macros in consts.h. + */ +/* clang-format off */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_ppc_qdata[2072] = { + /* -Q */ + -3329, -3329, -3329, -3329, -3329, -3329, -3329, -3329, + /* Q */ + 3329, 3329, 3329, 3329, 3329, 3329, 3329, 3329, + /* round(2^26 / MLKEM_Q), Barrett reduction constant */ + 20159, 20159, 20159, 20159, 20159, 20159, 20159, 20159, + /* N^-1 in Montgomery form */ + 512, 512, 512, 512, 512, 512, 512, 512, + /* Barrett twist of N^-1 */ + 5040, 5040, 5040, 5040, 5040, 5040, 5040, 5040, + /* 2^16 mod MLKEM_Q (R), Barrett tomont factor */ + -1044, -1044, -1044, -1044, -1044, -1044, -1044, -1044, + /* Barrett twist of R for poly_tomont */ + -10276, -10276, -10276, -10276, -10276, -10276, -10276, -10276, + /* Twiddle factors for the forward NTT */ + -1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600, + -749, -749, -749, -749, -749, -749, -749, -749, + -40, -40, -40, -40, -40, -40, -40, -40, + -687, -687, -687, -687, -687, -687, -687, -687, + 630, 630, 630, 630, 630, 630, 630, 630, + -1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432, + 848, 848, 848, 848, 848, 848, 848, 848, + 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + -1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410, + 193, 193, 193, 193, 193, 193, 193, 193, + 797, 797, 797, 797, 797, 797, 797, 797, + -543, -543, -543, -543, -543, -543, -543, -543, + -69, -69, -69, -69, -69, -69, -69, -69, + 569, 569, 569, 569, 569, 569, 569, 569, + -1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583, + 296, 296, 296, 296, 296, 296, 296, 296, + -882, -882, -882, -882, -882, -882, -882, -882, + 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339, + 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, + -283, -283, -283, -283, -283, -283, -283, -283, + 56, 56, 56, 56, 56, 56, 56, 56, + -1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089, + 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, + 1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426, + -1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235, + 535, 535, 535, 535, 535, 535, 535, 535, + -447, -447, -447, -447, -447, -447, -447, -447, + -936, -936, -936, -936, -936, -936, -936, -936, + -450, -450, -450, -450, -450, -450, -450, -450, + -1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355, + 821, 821, 821, 821, 821, 821, 821, 821, + 331, 331, 331, 331, 289, 289, 289, 289, + -1573, -1573, -1573, -1573, -76, -76, -76, -76, + -1025, -1025, -1025, -1025, 1197, 1197, 1197, 1197, + -1274, -1274, -1274, -1274, -1052, -1052, -1052, -1052, + -1352, -1352, -1352, -1352, 650, 650, 650, 650, + 632, 632, 632, 632, -816, -816, -816, -816, + 33, 33, 33, 33, -464, -464, -464, -464, + -1414, -1414, -1414, -1414, 1320, 1320, 1320, 1320, + 1435, 1435, 1435, 1435, -1010, -1010, -1010, -1010, + 452, 452, 452, 452, 807, 807, 807, 807, + -461, -461, -461, -461, 1438, 1438, 1438, 1438, + -927, -927, -927, -927, 1534, 1534, 1534, 1534, + -712, -712, -712, -712, -682, -682, -682, -682, + 648, 648, 648, 648, 1481, 1481, 1481, 1481, + -219, -219, -219, -219, -855, -855, -855, -855, + 910, 910, 910, 910, 1227, 1227, 1227, 1227, + 583, 583, 17, 17, -680, -680, -568, -568, + -1041, -1041, 1637, 1637, 1100, 1100, 723, 723, + -48, -48, 1409, 1409, 233, 233, -667, -667, + -314, -314, 756, 756, -279, -279, -1173, -1173, + -540, -540, -1626, -1626, -1540, -1540, 1651, 1651, + 1461, 1461, -1482, -1482, -642, -642, 952, 952, + -892, -892, 939, 939, -941, -941, -1021, -1021, + 268, 268, 733, 733, 641, 641, -992, -992, + -1292, -1292, 1584, 1584, -109, -109, -1031, -1031, + -1239, -1239, 375, 375, 1645, 1645, -780, -780, + -556, -556, 1063, 1063, 757, 757, 319, 319, + -863, -863, -1230, -1230, -735, -735, 561, 561, + 403, 403, -525, -525, 1026, 1026, 1092, 1092, + -554, -554, 1143, 1143, 886, 886, -1179, -1179, + -1455, -1455, -1607, -1607, 1029, 1029, 1212, 1212, + 885, 885, -1219, -1219, -1175, -1175, -394, -394, + /* Twiddle factors for the inverse NTT */ + -394, -394, -1175, -1175, -1219, -1219, 885, 885, + 1212, 1212, 1029, 1029, -1607, -1607, -1455, -1455, + -1179, -1179, 886, 886, 1143, 1143, -554, -554, + 1092, 1092, 1026, 1026, -525, -525, 403, 403, + 561, 561, -735, -735, -1230, -1230, -863, -863, + 319, 319, 757, 757, 1063, 1063, -556, -556, + -780, -780, 1645, 1645, 375, 375, -1239, -1239, + -1031, -1031, -109, -109, 1584, 1584, -1292, -1292, + -992, -992, 641, 641, 733, 733, 268, 268, + -1021, -1021, -941, -941, 939, 939, -892, -892, + 952, 952, -642, -642, -1482, -1482, 1461, 1461, + 1651, 1651, -1540, -1540, -1626, -1626, -540, -540, + -1173, -1173, -279, -279, 756, 756, -314, -314, + -667, -667, 233, 233, 1409, 1409, -48, -48, + 723, 723, 1100, 1100, 1637, 1637, -1041, -1041, + -568, -568, -680, -680, 17, 17, 583, 583, + 1227, 1227, 1227, 1227, 910, 910, 910, 910, + -855, -855, -855, -855, -219, -219, -219, -219, + 1481, 1481, 1481, 1481, 648, 648, 648, 648, + -682, -682, -682, -682, -712, -712, -712, -712, + 1534, 1534, 1534, 1534, -927, -927, -927, -927, + 1438, 1438, 1438, 1438, -461, -461, -461, -461, + 807, 807, 807, 807, 452, 452, 452, 452, + -1010, -1010, -1010, -1010, 1435, 1435, 1435, 1435, + 1320, 1320, 1320, 1320, -1414, -1414, -1414, -1414, + -464, -464, -464, -464, 33, 33, 33, 33, + -816, -816, -816, -816, 632, 632, 632, 632, + 650, 650, 650, 650, -1352, -1352, -1352, -1352, + -1052, -1052, -1052, -1052, -1274, -1274, -1274, -1274, + 1197, 1197, 1197, 1197, -1025, -1025, -1025, -1025, + -76, -76, -76, -76, -1573, -1573, -1573, -1573, + 289, 289, 289, 289, 331, 331, 331, 331, + 821, 821, 821, 821, 821, 821, 821, 821, + -1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355, + -450, -450, -450, -450, -450, -450, -450, -450, + -936, -936, -936, -936, -936, -936, -936, -936, + -447, -447, -447, -447, -447, -447, -447, -447, + 535, 535, 535, 535, 535, 535, 535, 535, + -1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235, + 1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426, + 1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333, + -1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089, + 56, 56, 56, 56, 56, 56, 56, 56, + -283, -283, -283, -283, -283, -283, -283, -283, + 1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476, + 1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339, + -882, -882, -882, -882, -882, -882, -882, -882, + 296, 296, 296, 296, 296, 296, 296, 296, + -1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583, + 569, 569, 569, 569, 569, 569, 569, 569, + -69, -69, -69, -69, -69, -69, -69, -69, + -543, -543, -543, -543, -543, -543, -543, -543, + 797, 797, 797, 797, 797, 797, 797, 797, + 193, 193, 193, 193, 193, 193, 193, 193, + -1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410, + 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + 848, 848, 848, 848, 848, 848, 848, 848, + -1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432, + 630, 630, 630, 630, 630, 630, 630, 630, + -687, -687, -687, -687, -687, -687, -687, -687, + -40, -40, -40, -40, -40, -40, -40, -40, + -749, -749, -749, -749, -749, -749, -749, -749, + -1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600, + /* Twisted twiddle factors for the forward NTT */ + -15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749, + -7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373, + -394, -394, -394, -394, -394, -394, -394, -394, + -6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762, + 6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201, + -14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095, + 8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347, + 10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453, + -13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879, + 1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900, + 7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845, + -5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345, + -679, -679, -679, -679, -679, -679, -679, -679, + 5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601, + -15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582, + 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914, + -8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682, + 13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180, + 14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529, + -2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786, + 551, 551, 551, 551, 551, 551, 551, 551, + -10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719, + 13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121, + 14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036, + -12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156, + 5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266, + -4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400, + -9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213, + -4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429, + -13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338, + 8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081, + 3258, 3258, 3258, 3258, 2845, 2845, 2845, 2845, + -15483, -15483, -15483, -15483, -748, -748, -748, -748, + -10089, -10089, -10089, -10089, 11782, 11782, 11782, 11782, + -12540, -12540, -12540, -12540, -10355, -10355, -10355, -10355, + -13308, -13308, -13308, -13308, 6398, 6398, 6398, 6398, + 6221, 6221, 6221, 6221, -8032, -8032, -8032, -8032, + 325, 325, 325, 325, -4567, -4567, -4567, -4567, + -13918, -13918, -13918, -13918, 12993, 12993, 12993, 12993, + 14125, 14125, 14125, 14125, -9942, -9942, -9942, -9942, + 4449, 4449, 4449, 4449, 7943, 7943, 7943, 7943, + -4538, -4538, -4538, -4538, 14155, 14155, 14155, 14155, + -9125, -9125, -9125, -9125, 15099, 15099, 15099, 15099, + -7008, -7008, -7008, -7008, -6713, -6713, -6713, -6713, + 6378, 6378, 6378, 6378, 14578, 14578, 14578, 14578, + -2156, -2156, -2156, -2156, -8416, -8416, -8416, -8416, + 8957, 8957, 8957, 8957, 12078, 12078, 12078, 12078, + 5739, 5739, 167, 167, -6693, -6693, -5591, -5591, + -10247, -10247, 16113, 16113, 10828, 10828, 7117, 7117, + -472, -472, 13869, 13869, 2293, 2293, -6565, -6565, + -3091, -3091, 7441, 7441, -2746, -2746, -11546, -11546, + -5315, -5315, -16005, -16005, -15159, -15159, 16251, 16251, + 14381, 14381, -14588, -14588, -6319, -6319, 9371, 9371, + -8780, -8780, 9243, 9243, -9262, -9262, -10050, -10050, + 2638, 2638, 7215, 7215, 6309, 6309, -9764, -9764, + -12717, -12717, 15592, 15592, -1073, -1073, -10148, -10148, + -12196, -12196, 3691, 3691, 16192, 16192, -7678, -7678, + -5473, -5473, 10463, 10463, 7451, 7451, 3140, 3140, + -8495, -8495, -12107, -12107, -7235, -7235, 5522, 5522, + 3967, 3967, -5168, -5168, 10099, 10099, 10749, 10749, + -5453, -5453, 11251, 11251, 8721, 8721, -11605, -11605, + -14322, -14322, -15818, -15818, 10129, 10129, 11930, 11930, + 8711, 8711, -11999, -11999, -11566, -11566, -3878, -3878, + /* Twisted twiddle factors for the inverse NTT */ + -3878, -3878, -11566, -11566, -11999, -11999, 8711, 8711, + 11930, 11930, 10129, 10129, -15818, -15818, -14322, -14322, + -11605, -11605, 8721, 8721, 11251, 11251, -5453, -5453, + 10749, 10749, 10099, 10099, -5168, -5168, 3967, 3967, + 5522, 5522, -7235, -7235, -12107, -12107, -8495, -8495, + 3140, 3140, 7451, 7451, 10463, 10463, -5473, -5473, + -7678, -7678, 16192, 16192, 3691, 3691, -12196, -12196, + -10148, -10148, -1073, -1073, 15592, 15592, -12717, -12717, + -9764, -9764, 6309, 6309, 7215, 7215, 2638, 2638, + -10050, -10050, -9262, -9262, 9243, 9243, -8780, -8780, + 9371, 9371, -6319, -6319, -14588, -14588, 14381, 14381, + 16251, 16251, -15159, -15159, -16005, -16005, -5315, -5315, + -11546, -11546, -2746, -2746, 7441, 7441, -3091, -3091, + -6565, -6565, 2293, 2293, 13869, 13869, -472, -472, + 7117, 7117, 10828, 10828, 16113, 16113, -10247, -10247, + -5591, -5591, -6693, -6693, 167, 167, 5739, 5739, + 12078, 12078, 12078, 12078, 8957, 8957, 8957, 8957, + -8416, -8416, -8416, -8416, -2156, -2156, -2156, -2156, + 14578, 14578, 14578, 14578, 6378, 6378, 6378, 6378, + -6713, -6713, -6713, -6713, -7008, -7008, -7008, -7008, + 15099, 15099, 15099, 15099, -9125, -9125, -9125, -9125, + 14155, 14155, 14155, 14155, -4538, -4538, -4538, -4538, + 7943, 7943, 7943, 7943, 4449, 4449, 4449, 4449, + -9942, -9942, -9942, -9942, 14125, 14125, 14125, 14125, + 12993, 12993, 12993, 12993, -13918, -13918, -13918, -13918, + -4567, -4567, -4567, -4567, 325, 325, 325, 325, + -8032, -8032, -8032, -8032, 6221, 6221, 6221, 6221, + 6398, 6398, 6398, 6398, -13308, -13308, -13308, -13308, + -10355, -10355, -10355, -10355, -12540, -12540, -12540, -12540, + 11782, 11782, 11782, 11782, -10089, -10089, -10089, -10089, + -748, -748, -748, -748, -15483, -15483, -15483, -15483, + 2845, 2845, 2845, 2845, 3258, 3258, 3258, 3258, + 8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081, + -13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338, + -4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429, + -9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213, + -4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400, + 5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266, + -12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156, + 14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036, + 13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121, + -10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719, + 551, 551, 551, 551, 551, 551, 551, 551, + -2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786, + 14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529, + 13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180, + -8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682, + 2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914, + -15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582, + 5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601, + -679, -679, -679, -679, -679, -679, -679, -679, + -5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345, + 7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845, + 1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900, + -13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879, + 10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453, + 8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347, + -14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095, + 6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201, + -6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762, + -394, -394, -394, -394, -394, -394, -394, -394, + -7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373, + -15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749, +}; +/* clang-format on */ + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED && __POWER8_VECTOR__ */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h new file mode 100644 index 0000000000..acac168b60 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/consts.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#define MLK_NATIVE_PPC64LE_SRC_CONSTS_H +#include "../../../common.h" + +/* Byte offsets into the constant table */ +#define MLK_PPC_NQ_OFFSET 0 +#define MLK_PPC_Q_OFFSET 16 +#define MLK_PPC_C20159_OFFSET 32 +#define MLK_PPC_N_INV_OFFSET 48 +#define MLK_PPC_N_INV_TW_OFFSET 64 +#define MLK_PPC_TOMONT_OFFSET 80 +#define MLK_PPC_TOMONT_TW_OFFSET 96 +#define MLK_PPC_ZETA_NTT_OFFSET 112 +#define MLK_PPC_ZETA_INTT_OFFSET 1120 +#define MLK_PPC_ZETA_NTT_TW_OFFSET 2128 +#define MLK_PPC_ZETA_INTT_TW_OFFSET 3136 + +#ifndef __ASSEMBLER__ +#define mlk_ppc_qdata MLK_NAMESPACE(ppc_qdata) +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_ppc_qdata[2072]; +#endif + +#endif /* !MLK_NATIVE_PPC64LE_SRC_CONSTS_H */ diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S new file mode 100644 index 0000000000..cd6929f6ac --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_intt_ppc_asm.S @@ -0,0 +1,3246 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) +/*yaml + Name: intt_ppc_asm + Description: PowerPC64 VSX inverse NTT + Signature: void mlk_intt_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_intt_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(intt_ppc_asm) +MLK_ASM_FN_SYMBOL(intt_ppc_asm) + + stdu 1, -352(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + stxvd2x 52, 10, 1 + stxvd2x 53, 11, 1 + stxvd2x 54, 12, 1 + stxvd2x 55, 14, 1 + stxvd2x 56, 15, 1 + stxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + stxvd2x 58, 10, 1 + stxvd2x 59, 11, 1 + stxvd2x 60, 12, 1 + stxvd2x 61, 14, 1 + stxvd2x 62, 15, 1 + stxvd2x 63, 16, 1 + lxvd2x 0, 0, 4 + xxlxor 35, 35, 35 + xxlor 3, 35, 35 + li 10, 16 + li 11, 32 + lxvd2x 6, 10, 4 + lxvd2x 32, 11, 4 + vspltisw 8, 13 + vadduwm 8, 8, 8 + xxlor 8, 40, 40 + vspltisw 9, 1 + vsubuwm 10, 8, 9 + vslw 9, 9, 10 + xxlor 7, 41, 41 + li 10, 16 + li 11, 32 + li 12, 48 + li 15, 64 + li 16, 80 + li 17, 96 + li 18, 112 + addi 14, 4, 48 + lvx 10, 0, 14 + addi 14, 4, 64 + lvx 11, 0, 14 + li 8, 4 + mtctr 8 + xxlor 37, 0, 0 + +mlk_intt_ppc_asm_Loopf: + lxvd2x 57, 0, 3 + lxvd2x 58, 10, 3 + lxvd2x 62, 11, 3 + lxvd2x 63, 12, 3 + addi 3, 3, 64 + vmhraddshs 14, 25, 11, 3 + vmhraddshs 19, 26, 11, 3 + vmhraddshs 24, 30, 11, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 6, 25, 10, 3 + vmladduhm 7, 26, 10, 3 + vmladduhm 8, 30, 10, 3 + vmladduhm 9, 31, 10, 3 + vmladduhm 6, 14, 5, 6 + vmladduhm 7, 19, 5, 7 + vmladduhm 8, 24, 5, 8 + vmladduhm 9, 29, 5, 9 + lxvd2x 57, 0, 3 + lxvd2x 58, 10, 3 + lxvd2x 62, 11, 3 + lxvd2x 63, 12, 3 + addi 3, 3, 64 + vmhraddshs 14, 25, 11, 3 + vmhraddshs 19, 26, 11, 3 + vmhraddshs 24, 30, 11, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + addi 3, 3, -128 + stxvd2x 38, 0, 3 + stxvd2x 39, 10, 3 + stxvd2x 40, 11, 3 + stxvd2x 41, 12, 3 + stxvd2x 45, 15, 3 + stxvd2x 50, 16, 3 + stxvd2x 55, 17, 3 + stxvd2x 60, 18, 3 + addi 3, 3, 128 + bdnz mlk_intt_ppc_asm_Loopf + addi 3, 3, -512 + nop + nop + ori 2, 2, 0 + addi 14, 4, 1120 + addi 22, 4, 3136 + li 7, 4 + mr 5, 3 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 8, 25, 26 + vmrgow 21, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 12, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 16, 25, 26 + vmrgow 23, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 20, 25, 26 + vmrgow 24, 25, 26 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + vmrgew 10, 13, 14 + vmrgow 11, 13, 14 + vmrgew 12, 18, 19 + vmrgow 13, 18, 19 + vmrgew 14, 23, 24 + vmrgow 15, 23, 24 + vmrgew 16, 28, 29 + vmrgow 17, 28, 29 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + mr 5, 3 + li 7, 8 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + lxvd2x 10, 0, 5 + lxvd2x 11, 10, 5 + xxmrgld 40, 11, 10 + xxmrghd 53, 11, 10 + lxvd2x 10, 11, 5 + lxvd2x 11, 12, 5 + xxmrgld 44, 11, 10 + xxmrghd 54, 11, 10 + lxvd2x 10, 15, 5 + lxvd2x 11, 16, 5 + xxmrgld 48, 11, 10 + xxmrghd 55, 11, 10 + lxvd2x 10, 17, 5 + lxvd2x 11, 18, 5 + xxmrgld 52, 11, 10 + xxmrghd 56, 11, 10 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + xxlor 10, 36, 36 + xxlor 11, 41, 41 + xxlor 12, 45, 45 + xxlor 13, 49, 49 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + xxlor 46, 10, 10 + xxlor 51, 11, 11 + xxlor 56, 12, 12 + xxlor 61, 13, 13 + xxmrgld 42, 46, 45 + xxmrghd 43, 46, 45 + xxmrgld 44, 51, 50 + xxmrghd 45, 51, 50 + xxmrgld 46, 56, 55 + xxmrghd 47, 56, 55 + xxmrgld 48, 61, 60 + xxmrghd 49, 61, 60 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + nop + ori 2, 2, 0 + li 7, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 32 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, -64 + addi 22, 22, -64 + li 9, 16 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, -64 + addi 22, 22, -64 + li 9, 272 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + li 8, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 8, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 44, 0, 22 + lxvd2x 52, 8, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 20, 3 + vmhraddshs 24, 30, 6, 3 + vmhraddshs 29, 31, 11, 3 + vmladduhm 13, 25, 7, 3 + vmladduhm 18, 26, 8, 3 + vmladduhm 23, 30, 9, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + nop + nop + ori 2, 2, 0 + li 7, 128 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 320 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + addi 14, 14, 16 + addi 22, 22, 16 + nop + nop + ori 2, 2, 0 + li 7, 256 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 9, 192 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 40, 3, 10 + lxvd2x 44, 3, 17 + lxvd2x 48, 3, 19 + lxvd2x 52, 3, 21 + lxvd2x 53, 3, 9 + lxvd2x 54, 3, 16 + lxvd2x 55, 3, 18 + lxvd2x 56, 3, 20 + vsubuhm 25, 8, 21 + vsubuhm 26, 12, 22 + vsubuhm 30, 16, 23 + vsubuhm 31, 20, 24 + vadduhm 8, 8, 21 + vadduhm 12, 12, 22 + vadduhm 16, 16, 23 + vadduhm 20, 20, 24 + vxor 7, 7, 7 + xxlor 35, 6, 6 + xxlor 33, 7, 7 + xxlor 34, 8, 8 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 36, 3, 9 + stxvd2x 41, 3, 16 + stxvd2x 45, 3, 18 + stxvd2x 49, 3, 20 + xxlor 37, 0, 0 + xxlor 35, 3, 3 + lvx 10, 0, 14 + lvx 12, 0, 22 + vmhraddshs 14, 25, 12, 3 + vmhraddshs 19, 26, 12, 3 + vmhraddshs 24, 30, 12, 3 + vmhraddshs 29, 31, 12, 3 + vmladduhm 13, 25, 10, 3 + vmladduhm 18, 26, 10, 3 + vmladduhm 23, 30, 10, 3 + vmladduhm 28, 31, 10, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 28, 29, 5, 28 + stxvd2x 45, 3, 10 + stxvd2x 50, 3, 17 + stxvd2x 55, 3, 19 + stxvd2x 60, 3, 21 + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + lxvd2x 52, 10, 1 + lxvd2x 53, 11, 1 + lxvd2x 54, 12, 1 + lxvd2x 55, 14, 1 + lxvd2x 56, 15, 1 + lxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + lxvd2x 58, 10, 1 + lxvd2x 59, 11, 1 + lxvd2x 60, 12, 1 + lxvd2x 61, 14, 1 + lxvd2x 62, 15, 1 + lxvd2x 63, 16, 1 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + mtlr 0 + addi 1, 1, 352 + blr + +MLK_ASM_FN_SIZE(intt_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S new file mode 100644 index 0000000000..09ec84a966 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_ntt_ppc_asm.S @@ -0,0 +1,1672 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: ntt_ppc_asm + Description: PowerPC64 VSX forward NTT + Signature: void mlk_ntt_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_ntt_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_ppc_asm) +MLK_ASM_FN_SYMBOL(ntt_ppc_asm) + + stdu 1, -352(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + stxvd2x 52, 10, 1 + stxvd2x 53, 11, 1 + stxvd2x 54, 12, 1 + stxvd2x 55, 14, 1 + stxvd2x 56, 15, 1 + stxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + stxvd2x 58, 10, 1 + stxvd2x 59, 11, 1 + stxvd2x 60, 12, 1 + stxvd2x 61, 14, 1 + stxvd2x 62, 15, 1 + stxvd2x 63, 16, 1 + lvx 5, 0, 4 + addi 14, 4, 112 + addi 22, 4, 2128 + vxor 3, 3, 3 + ori 2, 2, 0 + li 7, 256 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 192 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 128 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 64 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 320 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 64 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + lvx 10, 0, 14 + lvx 2, 0, 22 + addi 14, 14, 16 + addi 22, 22, 16 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 16 + addi 17, 10, 16 + addi 18, 16, 16 + addi 19, 17, 16 + addi 20, 18, 16 + addi 21, 19, 16 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 2, 3 + vmhraddshs 20, 18, 2, 3 + vmhraddshs 25, 23, 2, 3 + vmhraddshs 30, 28, 2, 3 + vmladduhm 13, 13, 10, 3 + vmladduhm 18, 18, 10, 3 + vmladduhm 23, 23, 10, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + nop + nop + ori 2, 2, 0 + li 7, 32 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 16 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 9, 272 + add 10, 7, 9 + addi 16, 9, 64 + addi 17, 10, 64 + addi 18, 16, 64 + addi 19, 17, 64 + addi 20, 18, 64 + addi 21, 19, 64 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + ori 2, 2, 0 + li 7, 16 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 0 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 128 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 256 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + li 9, 384 + add 10, 7, 9 + addi 16, 9, 32 + addi 17, 10, 32 + addi 18, 16, 32 + addi 19, 17, 32 + addi 20, 18, 32 + addi 21, 19, 32 + lxvd2x 45, 3, 10 + lxvd2x 50, 3, 17 + lxvd2x 55, 3, 19 + lxvd2x 60, 3, 21 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + lxvd2x 44, 3, 9 + lxvd2x 49, 3, 16 + lxvd2x 54, 3, 18 + lxvd2x 59, 3, 20 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + stxvd2x 47, 3, 9 + stxvd2x 48, 3, 10 + stxvd2x 52, 3, 16 + stxvd2x 53, 3, 17 + stxvd2x 57, 3, 18 + stxvd2x 58, 3, 19 + stxvd2x 62, 3, 20 + stxvd2x 63, 3, 21 + mr 5, 3 + li 7, 8 + li 10, 16 + li 11, 32 + li 12, 48 + li 15, 64 + li 16, 80 + li 17, 96 + li 18, 112 + nop + ori 2, 2, 0 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 1, 0, 5 + lxvd2x 2, 10, 5 + xxmrgld 45, 2, 1 + xxmrghd 44, 2, 1 + lxvd2x 3, 11, 5 + lxvd2x 4, 12, 5 + xxmrgld 50, 4, 3 + xxmrghd 49, 4, 3 + lxvd2x 1, 15, 5 + lxvd2x 2, 16, 5 + xxmrgld 55, 2, 1 + xxmrghd 54, 2, 1 + lxvd2x 3, 17, 5 + lxvd2x 4, 18, 5 + xxmrgld 60, 4, 3 + xxmrghd 59, 4, 3 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + xxmrgld 0, 47, 48 + xxmrghd 1, 47, 48 + xxmrgld 2, 52, 53 + xxmrghd 3, 52, 53 + xxmrgld 4, 57, 58 + xxmrghd 5, 57, 58 + xxmrgld 6, 62, 63 + xxmrghd 7, 62, 63 + stxvd2x 0, 0, 5 + stxvd2x 1, 10, 5 + stxvd2x 2, 11, 5 + stxvd2x 3, 12, 5 + stxvd2x 4, 15, 5 + stxvd2x 5, 16, 5 + stxvd2x 6, 17, 5 + stxvd2x 7, 18, 5 + addi 5, 5, 128 + mr 5, 3 + li 7, 4 + nop + ori 2, 2, 0 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 16 + li 11, 32 + li 12, 48 + lxvd2x 39, 0, 14 + lxvd2x 40, 10, 14 + lxvd2x 41, 11, 14 + lxvd2x 42, 12, 14 + lxvd2x 32, 0, 22 + lxvd2x 33, 10, 22 + lxvd2x 38, 11, 22 + lxvd2x 43, 12, 22 + addi 14, 14, 64 + addi 22, 22, 64 + lxvd2x 57, 0, 5 + lxvd2x 58, 10, 5 + vmrgew 13, 25, 26 + vmrgow 12, 25, 26 + lxvd2x 57, 11, 5 + lxvd2x 58, 12, 5 + vmrgew 18, 25, 26 + vmrgow 17, 25, 26 + lxvd2x 57, 15, 5 + lxvd2x 58, 16, 5 + vmrgew 23, 25, 26 + vmrgow 22, 25, 26 + lxvd2x 57, 17, 5 + lxvd2x 58, 18, 5 + vmrgew 28, 25, 26 + vmrgow 27, 25, 26 + vmhraddshs 15, 13, 0, 3 + vmhraddshs 20, 18, 1, 3 + vmhraddshs 25, 23, 6, 3 + vmhraddshs 30, 28, 11, 3 + vmladduhm 13, 13, 7, 3 + vmladduhm 18, 18, 8, 3 + vmladduhm 23, 23, 9, 3 + vmladduhm 28, 28, 10, 3 + vmladduhm 13, 15, 5, 13 + vmladduhm 18, 20, 5, 18 + vmladduhm 23, 25, 5, 23 + vmladduhm 28, 30, 5, 28 + vsubuhm 16, 12, 13 + vadduhm 15, 13, 12 + vsubuhm 21, 17, 18 + vadduhm 20, 18, 17 + vsubuhm 26, 22, 23 + vadduhm 25, 23, 22 + vsubuhm 31, 27, 28 + vadduhm 30, 28, 27 + vmrgew 10, 16, 15 + vmrgow 11, 16, 15 + vmrgew 12, 21, 20 + vmrgow 13, 21, 20 + vmrgew 14, 26, 25 + vmrgow 15, 26, 25 + vmrgew 16, 31, 30 + vmrgow 17, 31, 30 + stxvd2x 42, 0, 5 + stxvd2x 43, 10, 5 + stxvd2x 44, 11, 5 + stxvd2x 45, 12, 5 + stxvd2x 46, 15, 5 + stxvd2x 47, 16, 5 + stxvd2x 48, 17, 5 + stxvd2x 49, 18, 5 + addi 5, 5, 128 + li 10, 128 + li 11, 144 + li 12, 160 + li 14, 176 + li 15, 192 + li 16, 208 + lxvd2x 52, 10, 1 + lxvd2x 53, 11, 1 + lxvd2x 54, 12, 1 + lxvd2x 55, 14, 1 + lxvd2x 56, 15, 1 + lxvd2x 57, 16, 1 + li 10, 224 + li 11, 240 + li 12, 256 + li 14, 272 + li 15, 288 + li 16, 304 + lxvd2x 58, 10, 1 + lxvd2x 59, 11, 1 + lxvd2x 60, 12, 1 + lxvd2x 61, 14, 1 + lxvd2x 62, 15, 1 + lxvd2x 63, 16, 1 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + mtlr 0 + addi 1, 1, 352 + blr + +MLK_ASM_FN_SIZE(ntt_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S new file mode 100644 index 0000000000..3e430638d3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_poly_tomont_ppc_asm.S @@ -0,0 +1,316 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ + +/* + * Poly_tomont: Inplace conversion of all coefficients of a polynomial + * from normal domain to Montgomery domain + * + * Arguments:*r: pointer to input/output polynomial + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) && \ + !defined(MLK_CONFIG_NO_KEYPAIR_API) +/*yaml + Name: poly_tomont_ppc_asm + Description: PowerPC64 VSX conversion to Montgomery form + Signature: void mlk_poly_tomont_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_poly_tomont_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_tomont_ppc_asm) +MLK_ASM_FN_SYMBOL(poly_tomont_ppc_asm) + + stdu 1, -320(1) + mflr 0 + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + li 11, 208 + li 12, 224 + stxvd2x 52, 6, 1 + stxvd2x 53, 7, 1 + stxvd2x 54, 8, 1 + stxvd2x 55, 9, 1 + stxvd2x 56, 10, 1 + stxvd2x 57, 11, 1 + stxvd2x 58, 12, 1 + li 6, 240 + li 7, 256 + li 8, 272 + li 9, 288 + stxvd2x 59, 6, 1 + stxvd2x 60, 7, 1 + stxvd2x 61, 8, 1 + stxvd2x 62, 9, 1 + li 6, 0 + li 7, 80 + li 8, 96 + lxvd2x 37, 6, 4 + lxvd2x 32, 7, 4 + lxvd2x 34, 8, 4 + vxor 3, 3, 3 + li 4, -128 + li 5, -112 + li 6, -96 + li 7, -80 + li 8, -64 + li 9, -48 + li 10, -32 + li 11, -16 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 27, 14, 5, 13 + vmladduhm 28, 19, 5, 18 + vmladduhm 29, 24, 5, 23 + vmladduhm 30, 8, 5, 7 + lxvd2x 45, 0, 3 + addi 3, 3, 16 + lxvd2x 50, 0, 3 + addi 3, 3, 16 + lxvd2x 55, 0, 3 + addi 3, 3, 16 + lxvd2x 39, 0, 3 + addi 3, 3, 16 + vmhraddshs 14, 13, 2, 3 + vmhraddshs 19, 18, 2, 3 + vmhraddshs 24, 23, 2, 3 + vmhraddshs 8, 7, 2, 3 + vmladduhm 13, 13, 0, 3 + vmladduhm 18, 18, 0, 3 + vmladduhm 23, 23, 0, 3 + vmladduhm 7, 7, 0, 3 + vmladduhm 13, 14, 5, 13 + vmladduhm 18, 19, 5, 18 + vmladduhm 23, 24, 5, 23 + vmladduhm 7, 8, 5, 7 + stxvd2x 59, 4, 3 + stxvd2x 60, 5, 3 + stxvd2x 61, 6, 3 + stxvd2x 62, 7, 3 + stxvd2x 45, 8, 3 + stxvd2x 50, 9, 3 + stxvd2x 55, 10, 3 + stxvd2x 39, 11, 3 + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + li 11, 208 + li 12, 224 + lxvd2x 52, 6, 1 + lxvd2x 53, 7, 1 + lxvd2x 54, 8, 1 + lxvd2x 55, 9, 1 + lxvd2x 56, 10, 1 + lxvd2x 57, 11, 1 + lxvd2x 58, 12, 1 + li 6, 240 + li 7, 256 + li 8, 272 + li 9, 288 + lxvd2x 59, 6, 1 + lxvd2x 60, 7, 1 + lxvd2x 61, 8, 1 + lxvd2x 62, 9, 1 + mtlr 0 + addi 1, 1, 320 + blr + +MLK_ASM_FN_SIZE(poly_tomont_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ && !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S new file mode 100644 index 0000000000..e82376c74b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/ppc64le/src/mlkem_reduce_ppc_asm.S @@ -0,0 +1,731 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) IBM Corp. 2025, 2026 + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + * + * Written by Danny Tsen + */ +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_PPC64LE_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(__POWER8_VECTOR__) +/*yaml + Name: reduce_ppc_asm + Description: PowerPC64 VSX modular reduction + Signature: void mlk_reduce_ppc_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: powerpc64le + CallingConvention: ELFv2 + Features: [VSX] + r3: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + r4: + type: buffer + size_bytes: 4144 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (2072 x int16_t) +*/ + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/ppc64le/src/mlkem_reduce_ppc_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(reduce_ppc_asm) +MLK_ASM_FN_SYMBOL(reduce_ppc_asm) + + stdu 1, -224(1) + mflr 0 + std 14, 96(1) + std 15, 104(1) + std 16, 112(1) + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + stxvd2x 52, 6, 1 + stxvd2x 53, 7, 1 + stxvd2x 54, 8, 1 + stxvd2x 55, 9, 1 + stxvd2x 56, 10, 1 + vxor 7, 7, 7 + li 6, 16 + li 7, 32 + lxvd2x 35, 6, 4 + lxvd2x 32, 7, 4 + vspltisw 2, 13 + vadduwm 2, 2, 2 + vspltisw 4, 1 + vsubuwm 5, 2, 4 + vslw 1, 4, 5 + li 4, -128 + li 5, -112 + li 6, -96 + li 7, -80 + li 8, -64 + li 9, -48 + li 10, -32 + li 11, -16 + li 14, 16 + li 15, 32 + li 16, 48 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 21, 4, 3, 8 + vmladduhm 22, 9, 3, 12 + vmladduhm 23, 13, 3, 16 + vmladduhm 24, 17, 3, 20 + lxvd2x 40, 0, 3 + lxvd2x 44, 14, 3 + lxvd2x 48, 15, 3 + lxvd2x 52, 16, 3 + addi 3, 3, 64 + vmulosh 6, 8, 0 + vmulesh 5, 8, 0 + vmulosh 11, 12, 0 + vmulesh 10, 12, 0 + vmulosh 15, 16, 0 + vmulesh 14, 16, 0 + vmulosh 19, 20, 0 + vmulesh 18, 20, 0 + xxmrglw 36, 37, 38 + xxmrghw 37, 37, 38 + xxmrglw 41, 42, 43 + xxmrghw 42, 42, 43 + xxmrglw 45, 46, 47 + xxmrghw 46, 46, 47 + xxmrglw 49, 50, 51 + xxmrghw 50, 50, 51 + vadduwm 4, 4, 1 + vadduwm 5, 5, 1 + vadduwm 9, 9, 1 + vadduwm 10, 10, 1 + vadduwm 13, 13, 1 + vadduwm 14, 14, 1 + vadduwm 17, 17, 1 + vadduwm 18, 18, 1 + vsraw 4, 4, 2 + vsraw 5, 5, 2 + vsraw 9, 9, 2 + vsraw 10, 10, 2 + vsraw 13, 13, 2 + vsraw 14, 14, 2 + vsraw 17, 17, 2 + vsraw 18, 18, 2 + vpkuwum 4, 5, 4 + vsubuhm 4, 7, 4 + vpkuwum 9, 10, 9 + vsubuhm 9, 7, 9 + vpkuwum 13, 14, 13 + vsubuhm 13, 7, 13 + vpkuwum 17, 18, 17 + vsubuhm 17, 7, 17 + vmladduhm 4, 4, 3, 8 + vmladduhm 9, 9, 3, 12 + vmladduhm 13, 13, 3, 16 + vmladduhm 17, 17, 3, 20 + stxvd2x 53, 4, 3 + stxvd2x 54, 5, 3 + stxvd2x 55, 6, 3 + stxvd2x 56, 7, 3 + stxvd2x 36, 8, 3 + stxvd2x 41, 9, 3 + stxvd2x 45, 10, 3 + stxvd2x 49, 11, 3 + addi 3, 3, -512 + vxor 9, 9, 9 + vspltish 10, 15 + vmr 11, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + lxvd2x 44, 0, 3 + lxvd2x 45, 14, 3 + lxvd2x 46, 15, 3 + lxvd2x 47, 16, 3 + addi 3, 3, 64 + vsrh 1, 12, 10 + vsrh 0, 13, 10 + vsrh 3, 14, 10 + vsrh 2, 15, 10 + vadduhm 7, 12, 11 + vadduhm 8, 13, 11 + vadduhm 5, 14, 11 + vadduhm 6, 15, 11 + vcmpequh 1, 1, 9 + vcmpequh 0, 0, 9 + vcmpequh 3, 3, 9 + vcmpequh 2, 2, 9 + xxsel 33, 39, 44, 33 + xxsel 32, 40, 45, 32 + xxsel 35, 37, 46, 35 + xxsel 34, 38, 47, 34 + stxvd2x 35, 10, 3 + stxvd2x 34, 11, 3 + stxvd2x 33, 8, 3 + stxvd2x 32, 9, 3 + ld 14, 96(1) + ld 15, 104(1) + ld 16, 112(1) + li 6, 128 + li 7, 144 + li 8, 160 + li 9, 176 + li 10, 192 + lxvd2x 52, 6, 1 + lxvd2x 53, 7, 1 + lxvd2x 54, 8, 1 + lxvd2x 55, 9, 1 + lxvd2x 56, 10, 1 + mtlr 0 + addi 1, 1, 224 + blr + +MLK_ASM_FN_SIZE(reduce_ppc_asm) + +#endif /* MLK_ARITH_BACKEND_PPC64LE_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && __POWER8_VECTOR__ */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/riscv64/README.md b/src/mlkem-native/mlkem/src/native/riscv64/README.md new file mode 100644 index 0000000000..56ebe937e4 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/README.md @@ -0,0 +1,11 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +# RISC-V Vector Extension Backend + +This is an arithmetic backend for CPUs implementing the RISC-V Vector Extension. The backend is functional for all physical `VLEN`, but the NTT and inverse NTT are so far only implemented for VLEN=256, falling back to the default C implementations for other VLENs. + +## Requirements + +- RISC-V 64-bit architecture +- Vector extension (RVV) version 1.0 +- Standard "gc" extensions (integer and compressed instructions) diff --git a/src/mlkem-native/mlkem/src/native/riscv64/meta.h b/src/mlkem-native/mlkem/src/native/riscv64/meta.h new file mode 100644 index 0000000000..1dc637615b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/meta.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_RISCV64_META_H +#define MLK_NATIVE_RISCV64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_RISCV64 + +/* Set of primitives that this backend replaces */ +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_REJ_UNIFORM +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED + +#include "../../common.h" + +#if !defined(__ASSEMBLER__) +#include + +#include "../api.h" +#include "src/arith_native_riscv64.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + /* VLEN = 256 only for now */ + if (__riscv_vsetvlmax_e16m1() != 16) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_rv64v_poly_ntt(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + /* VLEN = 256 only for now */ + if (__riscv_vsetvlmax_e16m1() != 16) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_rv64v_poly_invntt_tomont(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + mlk_rv64v_poly_tomont(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + /* mlk_rv64v_rej_uniform requires buflen to be a multiple of 12 */ + if (buflen % 12 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + /* The cast from unsigned to signed integer is safe + * because the return value is <= len, which we asssume + * to be bound by 4096 and hence <= INT_MAX. */ + return (int)mlk_rv64v_rej_uniform(r, len, buf, buflen); +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + mlk_rv64v_poly_reduce(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + (void)x; /* not using the cache at the moment */ + (void)y; + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k2(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k3(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + (void)b_cache; + mlk_rv64v_poly_basemul_mont_add_k4(r, a, b); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_RISCV64_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h b/src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h new file mode 100644 index 0000000000..ab97ca9eb2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/arith_native_riscv64.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H +#define MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H + +#include "../../../common.h" + +#define mlk_rv64v_poly_ntt MLK_NAMESPACE(ntt_riscv64) +void mlk_rv64v_poly_ntt(int16_t *); + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_rv64v_poly_invntt_tomont MLK_NAMESPACE(intt_riscv64) +void mlk_rv64v_poly_invntt_tomont(int16_t *r); +#endif + +#define mlk_rv64v_poly_basemul_mont_add_k2 MLK_NAMESPACE(basemul_add_k2_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a, + const int16_t *b); + +#define mlk_rv64v_poly_basemul_mont_add_k3 MLK_NAMESPACE(basemul_add_k3_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a, + const int16_t *b); + +#define mlk_rv64v_poly_basemul_mont_add_k4 MLK_NAMESPACE(basemul_add_k4_riscv64) +void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a, + const int16_t *b); + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_rv64v_poly_tomont MLK_NAMESPACE(tomont_riscv64) +void mlk_rv64v_poly_tomont(int16_t *r); +#endif + +#define mlk_rv64v_poly_reduce MLK_NAMESPACE(reduce_riscv64) +void mlk_rv64v_poly_reduce(int16_t *r); + +#define mlk_rv64v_poly_add MLK_NAMESPACE(poly_add_riscv64) +void mlk_rv64v_poly_add(int16_t *r, const int16_t *a, const int16_t *b); + +#define mlk_rv64v_poly_sub MLK_NAMESPACE(poly_sub_riscv64) +void mlk_rv64v_poly_sub(int16_t *r, const int16_t *a, const int16_t *b); + +#define mlk_rv64v_rej_uniform MLK_NAMESPACE(rj_uniform_riscv64) +MLK_MUST_CHECK_RETURN_VALUE +unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len, + const uint8_t *buf, unsigned int buflen); + +#endif /* !MLK_NATIVE_RISCV64_SRC_ARITH_NATIVE_RISCV64_H */ diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c new file mode 100644 index 0000000000..46baa07998 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* NOTE: You can remove this file unless you compile with MLKEM_DEBUG. */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_RISCV64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && defined(MLKEM_DEBUG) + +#include +#include + +#include "../../../debug.h" +#include "rv64v_debug.h" + +#define MLK_DEBUG_ERROR_HEADER "[ERROR:%s:%04d] " + +/** + * Check whether values in a vint16m1_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (number of active elements). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + /* Allocate temporary array to store vector elements + * We use the maximum possible vector length to be safe */ + int16_t temp_array[64]; + + /* Store vector elements to temporary array for inspection */ + __riscv_vse16_v_i16m1(temp_array, vec, vl); + + /* Reuse existing array bounds checking function */ + mlk_debug_check_bounds(file, line, temp_array, (unsigned)vl, + lower_bound_exclusive, upper_bound_exclusive); +} + +/** + * Check whether values in a vint16m2_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (active elements per m1 half). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive) +{ + /* Allocate temporary array to store vector elements + * m2 vectors hold 2x the elements of m1 vectors */ + int16_t temp_array[2 * 64]; + + /* Store vector elements to temporary array for inspection */ + __riscv_vse16_v_i16m2(temp_array, vec, 2 * vl); + + /* Reuse existing array bounds checking function for all elements */ + mlk_debug_check_bounds(file, line, temp_array, (unsigned)(2 * vl), + lower_bound_exclusive, upper_bound_exclusive); +} + +#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + MLKEM_DEBUG */ + +MLK_EMPTY_CU(rv64v_debug) + +#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + MLKEM_DEBUG) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_DEBUG_ERROR_HEADER diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h new file mode 100644 index 0000000000..18d4b123f8 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_debug.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H +#define MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H + +#include "../../../debug.h" + +#include + +/************************************************* + * RISC-V Vector Bounds Assertion Macros + * + * These macros provide runtime bounds checking for RISC-V vector types + * vint16m1_t and vint16m2_t, following the same pattern as the scalar + * bounds assertions in debug.h + * + * The macros are only active when MLKEM_DEBUG is defined, otherwise they + * compile to no-ops for zero runtime overhead in release builds. + **************************************************/ + +#if defined(MLKEM_DEBUG) + +/** + * Check whether values in a vint16m1_t vector are within specified bounds. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (number of active elements). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds_int16m1 \ + MLK_NAMESPACE(mlkem_debug_check_bounds_int16m1) +void mlk_debug_check_bounds_int16m1(const char *file, int line, vint16m1_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive); + +/** + * Check whether values in a vint16m2_t vector are within specified bounds + * by splitting into m1 vectors. + * + * @param[in] file Filename. + * @param line Line number. + * @param vec RISC-V vector to be checked. + * @param vl Vector length (active elements per m1 half). + * @param lower_bound_exclusive Exclusive lower bound. + * @param upper_bound_exclusive Exclusive upper bound. + */ +#define mlk_debug_check_bounds_int16m2 \ + MLK_NAMESPACE(mlkem_debug_check_bounds_int16m2) +void mlk_debug_check_bounds_int16m2(const char *file, int line, vint16m2_t vec, + size_t vl, int lower_bound_exclusive, + int upper_bound_exclusive); + +/* Check bounds in vint16m1_t vector + * vec: RISC-V vector of type vint16m1_t + * vl: Vector length (number of active elements) + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + mlk_debug_check_bounds_int16m1(__FILE__, __LINE__, (vec), (vl), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in vint16m1_t vector + * vec: RISC-V vector of type vint16m1_t + * vl: Vector length (number of active elements) + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + mlk_assert_bound_int16m1((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd)) + +/* Check bounds in vint16m2_t vector + * vec: RISC-V vector of type vint16m2_t + * vl: Vector length (number of active elements per m1 half) + * value_lb: Inclusive lower value bound + * value_ub: Exclusive upper value bound */ +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + mlk_debug_check_bounds_int16m2(__FILE__, __LINE__, (vec), (vl), \ + (value_lb) - 1, (value_ub)) + +/* Check absolute bounds in vint16m2_t vector + * vec: RISC-V vector of type vint16m2_t + * vl: Vector length (number of active elements per m1 half) + * value_abs_bd: Exclusive absolute upper bound */ +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + mlk_assert_bound_int16m2((vec), (vl), (-(value_abs_bd) + 1), (value_abs_bd)) + +#elif defined(CBMC) + +/* For CBMC, we would need to implement vector bounds checking using CBMC + * primitives This is complex and would require extracting vector elements, so + * for now we provide empty implementations that could be extended later */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#else /* !MLKEM_DEBUG && CBMC */ + +/* When debugging is disabled, all assertions become no-ops */ +#define mlk_assert_bound_int16m1(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m1(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#define mlk_assert_bound_int16m2(vec, vl, value_lb, value_ub) \ + do \ + { \ + } while (0) + +#define mlk_assert_abs_bound_int16m2(vec, vl, value_abs_bd) \ + do \ + { \ + } while (0) + +#endif /* !MLKEM_DEBUG && !CBMC */ + +#endif /* !MLK_NATIVE_RISCV64_SRC_RV64V_DEBUG_H */ diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc new file mode 100644 index 0000000000..e24a9abe08 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_izetas.inc @@ -0,0 +1,27 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t izeta[] = { + -1044, 758, 1571, 205, 1275, -677, 1065, -448, -1628, -1522, 1460, + -958, -991, -996, 308, 108, 1517, 359, -411, 1542, 725, 1508, + -961, 398, -478, 870, 854, 1510, -794, 1278, 1530, 1185, -202, + -287, -608, -732, 951, 247, 1421, -107, 1659, 1187, -220, 874, + 1335, -1218, 136, 1215, -1422, -1493, -1017, 681, -830, 271, 90, + 853, -384, 1465, 1285, -1322, -610, -603, -1097, -817, -1468, 1474, + 130, 1602, -1469, -126, 1162, 1618, 75, 156, -329, -418, -349, + 872, -644, 1590, 1202, -962, -1458, 829, 666, 320, 8, -516, + -1119, 602, -1483, 777, 147, -1159, -778, 246, -182, -1577, -383, + -264, 1544, 282, -1491, 1293, -1653, -1574, 460, 291, 235, -177, + -587, -422, -622, 171, 1325, -573, -1015, 552, -652, -1223, -105, + -1550, -871, 1251, -843, -555, -430, 1103, +}; diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c new file mode 100644 index 0000000000..101bdab5dc --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_poly.c @@ -0,0 +1,797 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* === ML-KEM NTT using RISC-V Vector intrinstics */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_RISCV64) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include + +#include "arith_native_riscv64.h" +#include "rv64v_debug.h" + +/* Montgomery reduction constants */ +/* check-magic: -3327 == signed_mod(pow(MLKEM_Q,-1,2^16), 2^16) */ +#define MLK_RVV_QI -3327 + +/* check-magic: 2285 == unsigned_mod(2^16, MLKEM_Q) */ +#define MLK_RVV_MONT_R1 2285 + +/* check-magic: 1353 == pow(2, 32, MLKEM_Q) */ +#define MLK_RVV_MONT_R2 1353 + +/* check-magic: 1441 == pow(2,32-7,MLKEM_Q) */ +#define MLK_RVV_MONT_NR 1441 + +static MLK_INLINE vint16m1_t fq_redc(vint16m1_t rh, vint16m1_t rl, size_t vl) +{ + vint16m1_t t; + + t = __riscv_vmul_vx_i16m1(rl, MLK_RVV_QI, vl); /* t = l * Q^-1 */ + t = __riscv_vmulh_vx_i16m1(t, MLKEM_Q, vl); /* t = (t*Q) / R */ + t = __riscv_vsub_vv_i16m1(rh, t, vl); /* t = h - t */ + + return t; +} + +/* Narrowing reduction */ + +static MLK_INLINE vint16m1_t fq_redc2(vint32m2_t z, size_t vl) +{ + vint16m1_t t; + + t = __riscv_vmul_vx_i16m1(__riscv_vncvt_x_x_w_i16m1(z, vl), MLK_RVV_QI, + vl); /* t = l * Q^-1 */ + z = __riscv_vsub_vv_i32m2(z, __riscv_vwmul_vx_i32m2(t, MLKEM_Q, vl), + vl); /* x = (x - (t*Q)) */ + t = __riscv_vnsra_wx_i16m1(z, 16, vl); + + return t; +} + +/* Narrowing Barrett */ + +static MLK_INLINE vint16m1_t fq_barrett(vint16m1_t a, size_t vl) +{ + vint16m1_t t; + const int16_t v = ((1 << 26) + MLKEM_Q / 2) / MLKEM_Q; + + t = __riscv_vmulh_vx_i16m1(a, v, vl); + t = __riscv_vadd_vx_i16m1(t, 1 << (25 - 16), vl); + t = __riscv_vsra_vx_i16m1(t, 26 - 16, vl); + t = __riscv_vmul_vx_i16m1(t, MLKEM_Q, vl); + t = __riscv_vsub_vv_i16m1(a, t, vl); + + mlk_assert_abs_bound_int16m1(t, vl, MLKEM_Q_HALF); + return t; +} + +/* Conditionally add Q (if negative). + * + * Constant-time note: We deliberately avoid a `vmslt` + masked `vadd_mu` + * here. RVV 1.0 does not mandate mask-population-independent latency for + * masked ops, and Zvkt's DIEL guarantee does not cover the v0 mask + * register. Using an arithmetic-shift sign mask keeps all operands in the + * Zvkt-covered set and matches the idiom used in the portable C + * (mlk_ct_cmask_neg_i16, verify.h) and AArch64 + * (mlkem_poly_reduce_aarch64_asm.S) implementations. */ + +static MLK_INLINE vint16m1_t fq_cadd(vint16m1_t rx, size_t vl) +{ + vint16m1_t m; + + m = __riscv_vsra_vx_i16m1(rx, 15, vl); /* m = (x < 0) ? -1 : 0 */ + m = __riscv_vand_vx_i16m1(m, MLKEM_Q, vl); /* m = (x < 0) ? Q : 0 */ + rx = __riscv_vadd_vv_i16m1(rx, m, vl); /* x += m */ + return rx; +} + +/* Montgomery multiply: vector-vector */ + +static MLK_INLINE vint16m1_t fq_mul_vv(vint16m1_t rx, vint16m1_t ry, size_t vl) +{ + vint16m1_t rl, rh; + + rh = __riscv_vmulh_vv_i16m1(rx, ry, vl); /* h = (x * y) / R */ + rl = __riscv_vmul_vv_i16m1(rx, ry, vl); /* l = (x * y) % R */ + return fq_redc(rh, rl, vl); +} + +/* Montgomery multiply: vector-scalar */ + +static MLK_INLINE vint16m1_t fq_mul_vx(vint16m1_t rx, int16_t ry, size_t vl) +{ + vint16m1_t rl, rh; + + rh = __riscv_vmulh_vx_i16m1(rx, ry, vl); /* h = (x * y) / R */ + rl = __riscv_vmul_vx_i16m1(rx, ry, vl); /* l = (x * y) % R */ + return fq_redc(rh, rl, vl); +} + +/* full normalization */ + +static MLK_INLINE vint16m1_t fq_mulq_vx(vint16m1_t rx, int16_t ry, size_t vl) +{ + vint16m1_t result; + + result = fq_mul_vx(rx, ry, vl); + + mlk_assert_abs_bound_int16m1(result, vl, MLKEM_Q); + return result; +} + +/* create a permutation for swapping index bits a and b, a < b */ + +static vuint16m2_t bitswap_perm(unsigned a, unsigned b, size_t vl) +{ + const vuint16m2_t v2id = __riscv_vid_v_u16m2(vl); + + vuint16m2_t xa, xb; + xa = __riscv_vsrl_vx_u16m2(v2id, b - a, vl); + xa = __riscv_vxor_vv_u16m2(xa, v2id, vl); + xa = __riscv_vand_vx_u16m2(xa, (1 << a), vl); + xb = __riscv_vsll_vx_u16m2(xa, b - a, vl); + xa = __riscv_vxor_vv_u16m2(xa, xb, vl); + xa = __riscv_vxor_vv_u16m2(v2id, xa, vl); + return xa; +} + +/** + * Compute negacyclic number-theoretic transform (NTT) of a polynomial in + * place; input assumed to be in normal order, output in bitreversed order. + * + * @param[in,out] r Input/output polynomial. + */ + +/* Forward / Cooley-Tukey butterfly operation */ + +#define MLK_RVV_CT_BFLY_FX(u0, u1, ut, uc, vl, layer) \ + { \ + mlk_assert_abs_bound(&uc, 1, MLKEM_Q_HALF); \ + \ + ut = fq_mul_vx(u1, uc, vl); \ + mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \ + \ + u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \ + mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \ + } + +#define MLK_RVV_CT_BFLY_FV(u0, u1, ut, uc, vl, layer) \ + { \ + mlk_assert_abs_bound_int16m1(uc, vl, MLKEM_Q_HALF); \ + \ + ut = fq_mul_vv(u1, uc, vl); \ + mlk_assert_abs_bound_int16m1(ut, vl, MLKEM_Q); \ + \ + u1 = __riscv_vsub_vv_i16m1(u0, ut, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, ut, vl); \ + mlk_assert_abs_bound_int16m1(u0, vl, (layer + 1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(u1, vl, (layer + 1) * MLKEM_Q); \ + } + +static vint16m2_t mlk_rv64v_ntt2(vint16m2_t vp, vint16m1_t cz) +{ + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2); + const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2); + const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2); + + /* p1 = p8(p4(p2)) */ + const vuint16m2_t v2p1 = __riscv_vrgather_vv_u16m2( + __riscv_vrgather_vv_u16m2(v2p2, v2p4, vl2), v2p8, vl2); + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t cs8 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl); + const vuint16m1_t cs4 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl); + const vuint16m1_t cs2 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl); + + vint16m1_t vt, c0, t0, t1; + + /* swap 8 */ + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 5); + + /* swap 4 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 6); + + /* swap 2 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl); + MLK_RVV_CT_BFLY_FV(t0, t1, vt, c0, vl, 7); + + /* reorganize */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p1, vl2); + + return vp; +} + +/* Only for VLEN=256 for now */ +void mlk_rv64v_poly_ntt(int16_t *r) +{ +/* zetas can be compiled into vector constants; don't pass as a pointer */ +#include "rv64v_zetas.inc" + + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + vint16m1_t vt; + vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf; + + const vint16m1_t z0 = __riscv_vle16_v_i16m1(&zeta[0x00], vl); + const vint16m1_t z2 = __riscv_vle16_v_i16m1(&zeta[0x10], vl); + const vint16m1_t z4 = __riscv_vle16_v_i16m1(&zeta[0x20], vl); + const vint16m1_t z6 = __riscv_vle16_v_i16m1(&zeta[0x30], vl); + const vint16m1_t z8 = __riscv_vle16_v_i16m1(&zeta[0x40], vl); + const vint16m1_t za = __riscv_vle16_v_i16m1(&zeta[0x50], vl); + const vint16m1_t zc = __riscv_vle16_v_i16m1(&zeta[0x60], vl); + const vint16m1_t ze = __riscv_vle16_v_i16m1(&zeta[0x70], vl); + + v0 = __riscv_vle16_v_i16m1(&r[0x00], vl); + v1 = __riscv_vle16_v_i16m1(&r[0x10], vl); + v2 = __riscv_vle16_v_i16m1(&r[0x20], vl); + v3 = __riscv_vle16_v_i16m1(&r[0x30], vl); + v4 = __riscv_vle16_v_i16m1(&r[0x40], vl); + v5 = __riscv_vle16_v_i16m1(&r[0x50], vl); + v6 = __riscv_vle16_v_i16m1(&r[0x60], vl); + v7 = __riscv_vle16_v_i16m1(&r[0x70], vl); + v8 = __riscv_vle16_v_i16m1(&r[0x80], vl); + v9 = __riscv_vle16_v_i16m1(&r[0x90], vl); + va = __riscv_vle16_v_i16m1(&r[0xa0], vl); + vb = __riscv_vle16_v_i16m1(&r[0xb0], vl); + vc = __riscv_vle16_v_i16m1(&r[0xc0], vl); + vd = __riscv_vle16_v_i16m1(&r[0xd0], vl); + ve = __riscv_vle16_v_i16m1(&r[0xe0], vl); + vf = __riscv_vle16_v_i16m1(&r[0xf0], vl); + + MLK_RVV_CT_BFLY_FX(v0, v8, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v1, v9, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v2, va, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v3, vb, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v4, vc, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v5, vd, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v6, ve, vt, zeta[0x01], vl, 1); + MLK_RVV_CT_BFLY_FX(v7, vf, vt, zeta[0x01], vl, 1); + + MLK_RVV_CT_BFLY_FX(v0, v4, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v1, v5, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v2, v6, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v3, v7, vt, zeta[0x10], vl, 2); + MLK_RVV_CT_BFLY_FX(v8, vc, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(v9, vd, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(va, ve, vt, zeta[0x11], vl, 2); + MLK_RVV_CT_BFLY_FX(vb, vf, vt, zeta[0x11], vl, 2); + + MLK_RVV_CT_BFLY_FX(v0, v2, vt, zeta[0x20], vl, 3); + MLK_RVV_CT_BFLY_FX(v1, v3, vt, zeta[0x20], vl, 3); + MLK_RVV_CT_BFLY_FX(v4, v6, vt, zeta[0x21], vl, 3); + MLK_RVV_CT_BFLY_FX(v5, v7, vt, zeta[0x21], vl, 3); + MLK_RVV_CT_BFLY_FX(v8, va, vt, zeta[0x30], vl, 3); + MLK_RVV_CT_BFLY_FX(v9, vb, vt, zeta[0x30], vl, 3); + MLK_RVV_CT_BFLY_FX(vc, ve, vt, zeta[0x31], vl, 3); + MLK_RVV_CT_BFLY_FX(vd, vf, vt, zeta[0x31], vl, 3); + + MLK_RVV_CT_BFLY_FX(v0, v1, vt, zeta[0x40], vl, 4); + MLK_RVV_CT_BFLY_FX(v2, v3, vt, zeta[0x41], vl, 4); + MLK_RVV_CT_BFLY_FX(v4, v5, vt, zeta[0x50], vl, 4); + MLK_RVV_CT_BFLY_FX(v6, v7, vt, zeta[0x51], vl, 4); + MLK_RVV_CT_BFLY_FX(v8, v9, vt, zeta[0x60], vl, 4); + MLK_RVV_CT_BFLY_FX(va, vb, vt, zeta[0x61], vl, 4); + MLK_RVV_CT_BFLY_FX(vc, vd, vt, zeta[0x70], vl, 4); + MLK_RVV_CT_BFLY_FX(ve, vf, vt, zeta[0x71], vl, 4); + + __riscv_vse16_v_i16m2( + &r[0x00], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v0, v1), z0), vl2); + __riscv_vse16_v_i16m2( + &r[0x20], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v2, v3), z2), vl2); + __riscv_vse16_v_i16m2( + &r[0x40], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v4, v5), z4), vl2); + __riscv_vse16_v_i16m2( + &r[0x60], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v6, v7), z6), vl2); + __riscv_vse16_v_i16m2( + &r[0x80], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(v8, v9), z8), vl2); + __riscv_vse16_v_i16m2( + &r[0xa0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(va, vb), za), vl2); + __riscv_vse16_v_i16m2( + &r[0xc0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(vc, vd), zc), vl2); + __riscv_vse16_v_i16m2( + &r[0xe0], mlk_rv64v_ntt2(__riscv_vcreate_v_i16m1_i16m2(ve, vf), ze), vl2); +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reverse / Gentleman-Sande butterfly operation */ + +#define MLK_RVV_GS_BFLY_RX(u0, u1, ut, uc, vl) \ + { \ + ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \ + u1 = fq_mul_vx(ut, uc, vl); \ + } + +#define MLK_RVV_GS_BFLY_RV(u0, u1, ut, uc, vl) \ + { \ + ut = __riscv_vsub_vv_i16m1(u0, u1, vl); \ + u0 = __riscv_vadd_vv_i16m1(u0, u1, vl); \ + u1 = fq_mul_vv(ut, uc, vl); \ + } + +static vint16m2_t mlk_rv64v_intt2(vint16m2_t vp, vint16m1_t cz) +{ + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vuint16m2_t v2p8 = bitswap_perm(3, 4, vl2); + const vuint16m2_t v2p4 = bitswap_perm(2, 4, vl2); + const vuint16m2_t v2p2 = bitswap_perm(1, 4, vl2); + + /* p0 = p2(p4(p8)) */ + const vuint16m2_t v2p0 = __riscv_vrgather_vv_u16m2( + __riscv_vrgather_vv_u16m2(v2p8, v2p4, vl2), v2p2, vl2); + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t cs8 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 3, vl), 2, vl); + const vuint16m1_t cs4 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 2, vl), 2 + 2, vl); + const vuint16m1_t cs2 = + __riscv_vadd_vx_u16m1(__riscv_vsrl_vx_u16m1(vid, 1, vl), 2 + 2 + 4, vl); + + vint16m1_t t0, t1, c0, vt; + + /* initial permute */ + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p0, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + /* pre-scale */ + t0 = fq_mulq_vx(t0, MLK_RVV_MONT_NR, vl); + t1 = fq_mulq_vx(t1, MLK_RVV_MONT_NR, vl); + + /* absolute bounds: < t0 < q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + c0 = __riscv_vrgather_vv_i16m1(cz, cs2, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: < t0 < 2*q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, 2 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 2 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p2, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + c0 = __riscv_vrgather_vv_i16m1(cz, cs4, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: t0 < 4*q, t1 < q */ + mlk_assert_abs_bound_int16m1(t0, vl, 4 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 4 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p4, vl2); + t0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + t1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + c0 = __riscv_vrgather_vv_i16m1(cz, cs8, vl); + MLK_RVV_GS_BFLY_RV(t0, t1, vt, c0, vl); + + /* absolute bounds: < 8*q */ + mlk_assert_abs_bound_int16m1(t0, vl, 8 * MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + t0 = fq_mulq_vx(t0, MLK_RVV_MONT_R1, vl); + + /* absolute bounds: < q */ + mlk_assert_abs_bound_int16m1(t0, vl, MLKEM_Q); + mlk_assert_abs_bound_int16m1(t1, vl, MLKEM_Q); + + /* swap 8 */ + vp = __riscv_vcreate_v_i16m1_i16m2(t0, t1); + vp = __riscv_vrgatherei16_vv_i16m2(vp, v2p8, vl2); + + return vp; +} + +#define MLK_RV64V_ABS_BOUNDS16(vl, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, \ + vb, vc, vd, ve, vf, b0, b1, b2, b3, b4, b5, b6, \ + b7, b8, b9, ba, bb, bc, bd, be, bf) \ + do \ + { \ + mlk_assert_abs_bound_int16m1(v0, vl, (b0) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v1, vl, (b1) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v2, vl, (b2) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v3, vl, (b3) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v4, vl, (b4) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v5, vl, (b5) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v6, vl, (b6) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v7, vl, (b7) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v8, vl, (b8) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(v9, vl, (b9) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(va, vl, (ba) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vb, vl, (bb) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vc, vl, (bc) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vd, vl, (bd) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(ve, vl, (be) * MLKEM_Q); \ + mlk_assert_abs_bound_int16m1(vf, vl, (bf) * MLKEM_Q); \ + } while (0) + + +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place; input assumed to be in bitreversed order, output in + * normal order. + * + * @param[in,out] r Input/output polynomial. + */ +/* Only for VLEN=256 for now */ +void mlk_rv64v_poly_invntt_tomont(int16_t *r) +{ +/* zetas can be compiled into vector constants; don't pass as a pointer */ +#include "rv64v_izetas.inc" + + size_t vl = 16; /* We work with 256-bit vectors of 16x16-bit elements */ + size_t vl2 = 2 * vl; + + const vint16m1_t z0 = __riscv_vle16_v_i16m1(&izeta[0x00], vl); + const vint16m1_t z2 = __riscv_vle16_v_i16m1(&izeta[0x10], vl); + const vint16m1_t z4 = __riscv_vle16_v_i16m1(&izeta[0x20], vl); + const vint16m1_t z6 = __riscv_vle16_v_i16m1(&izeta[0x30], vl); + const vint16m1_t z8 = __riscv_vle16_v_i16m1(&izeta[0x40], vl); + const vint16m1_t za = __riscv_vle16_v_i16m1(&izeta[0x50], vl); + const vint16m1_t zc = __riscv_vle16_v_i16m1(&izeta[0x60], vl); + const vint16m1_t ze = __riscv_vle16_v_i16m1(&izeta[0x70], vl); + + vint16m1_t vt; + vint16m1_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf; + vint16m2_t vp; + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x00], vl2), z0); + v0 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v1 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x20], vl2), z2); + v2 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v3 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x40], vl2), z4); + v4 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v5 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x60], vl2), z6); + v6 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v7 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0x80], vl2), z8); + v8 = __riscv_vget_v_i16m2_i16m1(vp, 0); + v9 = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xa0], vl2), za); + va = __riscv_vget_v_i16m2_i16m1(vp, 0); + vb = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xc0], vl2), zc); + vc = __riscv_vget_v_i16m2_i16m1(vp, 0); + vd = __riscv_vget_v_i16m2_i16m1(vp, 1); + + vp = mlk_rv64v_intt2(__riscv_vle16_v_i16m2(&r[0xe0], vl2), ze); + ve = __riscv_vget_v_i16m2_i16m1(vp, 0); + vf = __riscv_vget_v_i16m2_i16m1(vp, 1); + + /* absolute bounds < q (see mlk_rv64v_intt2) */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + + MLK_RVV_GS_BFLY_RX(v0, v1, vt, izeta[0x40], vl); + MLK_RVV_GS_BFLY_RX(v2, v3, vt, izeta[0x41], vl); + MLK_RVV_GS_BFLY_RX(v4, v5, vt, izeta[0x50], vl); + MLK_RVV_GS_BFLY_RX(v6, v7, vt, izeta[0x51], vl); + MLK_RVV_GS_BFLY_RX(v8, v9, vt, izeta[0x60], vl); + MLK_RVV_GS_BFLY_RX(va, vb, vt, izeta[0x61], vl); + MLK_RVV_GS_BFLY_RX(vc, vd, vt, izeta[0x70], vl); + MLK_RVV_GS_BFLY_RX(ve, vf, vt, izeta[0x71], vl); + + /* absolute bounds: + * - v{0,2,4,6,8,a,c,e}: < 2*q + * - v{1,3,5,7,9,b,d,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1); + + MLK_RVV_GS_BFLY_RX(v0, v2, vt, izeta[0x20], vl); + MLK_RVV_GS_BFLY_RX(v1, v3, vt, izeta[0x20], vl); + MLK_RVV_GS_BFLY_RX(v4, v6, vt, izeta[0x21], vl); + MLK_RVV_GS_BFLY_RX(v5, v7, vt, izeta[0x21], vl); + MLK_RVV_GS_BFLY_RX(v8, va, vt, izeta[0x30], vl); + MLK_RVV_GS_BFLY_RX(v9, vb, vt, izeta[0x30], vl); + MLK_RVV_GS_BFLY_RX(vc, ve, vt, izeta[0x31], vl); + MLK_RVV_GS_BFLY_RX(vd, vf, vt, izeta[0x31], vl); + + /* absolute bounds: + * - v{0,4,8,c}: < 4*q + * - v{1,5,9,d}: < 2*q + * - v{2,3,6,7,a,b,e,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1, 4, 2, 1, 1); + + MLK_RVV_GS_BFLY_RX(v0, v4, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v1, v5, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v2, v6, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v3, v7, vt, izeta[0x10], vl); + MLK_RVV_GS_BFLY_RX(v8, vc, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(v9, vd, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(va, ve, vt, izeta[0x11], vl); + MLK_RVV_GS_BFLY_RX(vb, vf, vt, izeta[0x11], vl); + + /* absolute bounds: + * - v{0,8}: < 8*q + * - v{1,9}: < 4*q + * - v{2,3,a,b}: < 2*q + * - v{4,5,6,7,c,d,e,f}: < 1*q + */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 8, 4, 2, 2, 1, 1, 1, 1, 8, 4, 2, 2, 1, 1, 1, 1); + + /* Reduce v0, v8 to avoid overflow */ + v0 = fq_mulq_vx(v0, MLK_RVV_MONT_R1, vl); + v8 = fq_mulq_vx(v8, MLK_RVV_MONT_R1, vl); + + /* absolute bounds: < 4*q */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4); + + MLK_RVV_GS_BFLY_RX(v0, v8, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v1, v9, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v2, va, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v3, vb, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v4, vc, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v5, vd, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v6, ve, vt, izeta[0x01], vl); + MLK_RVV_GS_BFLY_RX(v7, vf, vt, izeta[0x01], vl); + + /* absolute bounds: < 8*q */ + MLK_RV64V_ABS_BOUNDS16(vl, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, va, vb, vc, vd, ve, vf, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8); + + __riscv_vse16_v_i16m1(&r[0x00], v0, vl); + __riscv_vse16_v_i16m1(&r[0x10], v1, vl); + __riscv_vse16_v_i16m1(&r[0x20], v2, vl); + __riscv_vse16_v_i16m1(&r[0x30], v3, vl); + __riscv_vse16_v_i16m1(&r[0x40], v4, vl); + __riscv_vse16_v_i16m1(&r[0x50], v5, vl); + __riscv_vse16_v_i16m1(&r[0x60], v6, vl); + __riscv_vse16_v_i16m1(&r[0x70], v7, vl); + __riscv_vse16_v_i16m1(&r[0x80], v8, vl); + __riscv_vse16_v_i16m1(&r[0x90], v9, vl); + __riscv_vse16_v_i16m1(&r[0xa0], va, vl); + __riscv_vse16_v_i16m1(&r[0xb0], vb, vl); + __riscv_vse16_v_i16m1(&r[0xc0], vc, vl); + __riscv_vse16_v_i16m1(&r[0xd0], vd, vl); + __riscv_vse16_v_i16m1(&r[0xe0], ve, vl); + __riscv_vse16_v_i16m1(&r[0xf0], vf, vl); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* ML-KEM's middle field GF(3329)[X]/(X^2) multiplication */ + +static MLK_INLINE void mlk_rv64v_poly_basemul_mont_add_k(int16_t *r, + const int16_t *a, + const int16_t *b, + unsigned kn) +{ +#include "rv64v_zetas_basemul.inc" + + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + size_t i, j; + + const vuint16m1_t sw0 = __riscv_vxor_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl); + const vbool16_t sb0 = __riscv_vmseq_vx_u16m1_b16( + __riscv_vand_vx_u16m1(__riscv_vid_v_u16m1(vl), 1, vl), 0, vl); + + vint16m1_t vt, vu; + vint32m2_t wa, wb, ws; + + for (i = 0; i < MLKEM_N; i += vl) + { + const vint16m1_t vz = __riscv_vle16_v_i16m1(&roots[i], vl); + + for (j = 0; j < kn; j += MLKEM_N) + { + vt = __riscv_vle16_v_i16m1(&a[i + j], vl); + vu = __riscv_vle16_v_i16m1(&b[i + j], vl); + + wa = __riscv_vwmul_vv_i32m2(vz, fq_mul_vv(vt, vu, vl), vl); + wb = __riscv_vwmul_vv_i32m2(vt, __riscv_vrgather_vv_i16m1(vu, sw0, vl), + vl); + + wa = + __riscv_vadd_vv_i32m2(wa, __riscv_vslidedown_vx_i32m2(wa, 1, vl), vl); + wb = __riscv_vadd_vv_i32m2(wb, __riscv_vslideup_vx_i32m2(wb, wb, 1, vl), + vl); + + wa = __riscv_vmerge_vvm_i32m2(wb, wa, sb0, vl); + + if (j == 0) + { + ws = wa; + } + else + { + ws = __riscv_vadd_vv_i32m2(ws, wa, vl); + } + } + /* the idea is to keep 32-bit intermediate result, reduce in the end */ + __riscv_vse16_v_i16m1(&r[i], fq_redc2(ws, vl), vl); + } +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +void mlk_rv64v_poly_basemul_mont_add_k2(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 2 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +void mlk_rv64v_poly_basemul_mont_add_k3(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 3 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +void mlk_rv64v_poly_basemul_mont_add_k4(int16_t *r, const int16_t *a, + const int16_t *b) +{ + mlk_rv64v_poly_basemul_mont_add_k(r, a, b, 4 * MLKEM_N); +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * @param[in,out] r Input/output polynomial. + */ +void mlk_rv64v_poly_tomont(int16_t *r) +{ + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + size_t i; + + for (i = 0; i < MLKEM_N; i += vl) + { + vint16m1_t vec = __riscv_vle16_v_i16m1(&r[i], vl); + vec = fq_mul_vx(vec, MLK_RVV_MONT_R2, vl); + __riscv_vse16_v_i16m1(&r[i], vec, vl); + } +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Apply Barrett reduction to all coefficients of a polynomial. For details + * of the Barrett reduction see the comments in poly.c. + * + * @param[in,out] r Input/output polynomial. + */ +void mlk_rv64v_poly_reduce(int16_t *r) +{ + size_t vl = __riscv_vsetvl_e16m1(MLKEM_N); + vint16m1_t vt; + size_t i; + + for (i = 0; i < MLKEM_N; i += vl) + { + vt = __riscv_vle16_v_i16m1(&r[i], vl); + vt = fq_barrett(vt, vl); + vt = fq_cadd(vt, vl); + __riscv_vse16_v_i16m1(&r[i], vt, vl); + } +} + +/* Run rejection sampling to get uniform random integers mod q. + * buflen must be a multiple of 12. */ + +unsigned int mlk_rv64v_rej_uniform(int16_t *r, unsigned int len, + const uint8_t *buf, unsigned int buflen) +{ + unsigned n, ctr, pos; + vuint16m1_t x, y; + vbool16_t lt; + + pos = 0; + ctr = 0; + + while (ctr < len && pos < buflen) + { + const unsigned vl = (unsigned)__riscv_vsetvl_e16m1((buflen - pos) * 8 / 12); + const unsigned vl23 = (vl * 24) / 32; + + const vuint16m1_t vid = __riscv_vid_v_u16m1(vl); + const vuint16m1_t srl12v = __riscv_vmul_vx_u16m1(vid, 12, vl); + const vuint16m1_t sel12v = __riscv_vsrl_vx_u16m1(srl12v, 4, vl); + const vuint16m1_t sll12v = __riscv_vsll_vx_u16m1(vid, 2, vl); + + /* Functionally, this loop is not necessary, but it avoids re-evaluating + * the VL too many times. In particular, in the first outer iteration, + * the inner loop will process the bulk of the data with fixed VL. */ + while (ctr < len && vl23 * 2 <= buflen - pos) + { + x = __riscv_vle16_v_u16m1((uint16_t *)&buf[pos], vl23); + pos += vl23 * 2; + x = __riscv_vrgather_vv_u16m1(x, sel12v, vl); + x = __riscv_vor_vv_u16m1( + __riscv_vsrl_vv_u16m1(x, srl12v, vl), + __riscv_vsll_vv_u16m1(__riscv_vslidedown(x, 1, vl), sll12v, vl), vl); + x = __riscv_vand_vx_u16m1(x, 0xFFF, vl); + + lt = __riscv_vmsltu_vx_u16m1_b16(x, MLKEM_Q, vl); + y = __riscv_vcompress_vm_u16m1(x, lt, vl); + n = (unsigned)__riscv_vcpop_m_b16(lt, vl); + + if (ctr + n > len) + { + n = len - ctr; + } + __riscv_vse16_v_u16m1((uint16_t *)&r[ctr], y, n); + ctr += n; + } + } + + return ctr; +} + +#else /* MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(rv64v_poly) + +#endif /* !(MLK_ARITH_BACKEND_RISCV64 && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLK_RVV_QI +#undef MLK_RVV_MONT_R1 +#undef MLK_RVV_MONT_R2 +#undef MLK_RVV_MONT_NR +#undef MLK_RVV_CT_BFLY_FX +#undef MLK_RVV_CT_BFLY_FV +#undef MLK_RVV_GS_BFLY_RX +#undef MLK_RVV_GS_BFLY_RV +#undef MLK_RV64V_ABS_BOUNDS16 diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc new file mode 100644 index 0000000000..9deb95900b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas.inc @@ -0,0 +1,27 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t zeta[] = { + -1044, -758, 573, -1325, 1223, 652, -552, 1015, -1103, 430, 555, + 843, -1251, 871, 1550, 105, -359, -1517, 264, 383, -1293, 1491, + -282, -1544, 422, 587, 177, -235, -291, -460, 1574, 1653, 1493, + 1422, -829, 1458, 516, -8, -320, -666, -246, 778, 1159, -147, + -777, 1483, -602, 1119, 287, 202, -1602, -130, -1618, -1162, 126, + 1469, -1590, 644, -872, 349, 418, 329, -156, -75, -171, 622, + -681, 1017, -853, -90, -271, 830, 817, 1097, 603, 610, 1322, + -1285, -1465, 384, 1577, 182, 732, 608, 107, -1421, -247, -951, + -1215, -136, 1218, -1335, -874, 220, -1187, -1659, 962, -1202, -1542, + 411, -398, 961, -1508, -725, -1185, -1530, -1278, 794, -1510, -854, + -870, 478, -1474, 1468, -205, -1571, 448, -1065, 677, -1275, -108, + -308, 996, 991, 958, -1460, 1522, 1628, +}; diff --git a/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc new file mode 100644 index 0000000000..ee4033c6ee --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/riscv64/src/rv64v_zetas_basemul.inc @@ -0,0 +1,39 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "arith_native_riscv64.h" + +const int16_t roots[] = { + -1044, -1103, -1044, 1103, -1044, 430, -1044, -430, -1044, 555, -1044, + -555, -1044, 843, -1044, -843, -1044, -1251, -1044, 1251, -1044, 871, + -1044, -871, -1044, 1550, -1044, -1550, -1044, 105, -1044, -105, -1044, + 422, -1044, -422, -1044, 587, -1044, -587, -1044, 177, -1044, -177, + -1044, -235, -1044, 235, -1044, -291, -1044, 291, -1044, -460, -1044, + 460, -1044, 1574, -1044, -1574, -1044, 1653, -1044, -1653, -1044, -246, + -1044, 246, -1044, 778, -1044, -778, -1044, 1159, -1044, -1159, -1044, + -147, -1044, 147, -1044, -777, -1044, 777, -1044, 1483, -1044, -1483, + -1044, -602, -1044, 602, -1044, 1119, -1044, -1119, -1044, -1590, -1044, + 1590, -1044, 644, -1044, -644, -1044, -872, -1044, 872, -1044, 349, + -1044, -349, -1044, 418, -1044, -418, -1044, 329, -1044, -329, -1044, + -156, -1044, 156, -1044, -75, -1044, 75, -1044, 817, -1044, -817, + -1044, 1097, -1044, -1097, -1044, 603, -1044, -603, -1044, 610, -1044, + -610, -1044, 1322, -1044, -1322, -1044, -1285, -1044, 1285, -1044, -1465, + -1044, 1465, -1044, 384, -1044, -384, -1044, -1215, -1044, 1215, -1044, + -136, -1044, 136, -1044, 1218, -1044, -1218, -1044, -1335, -1044, 1335, + -1044, -874, -1044, 874, -1044, 220, -1044, -220, -1044, -1187, -1044, + 1187, -1044, -1659, -1044, 1659, -1044, -1185, -1044, 1185, -1044, -1530, + -1044, 1530, -1044, -1278, -1044, 1278, -1044, 794, -1044, -794, -1044, + -1510, -1044, 1510, -1044, -854, -1044, 854, -1044, -870, -1044, 870, + -1044, 478, -1044, -478, -1044, -108, -1044, 108, -1044, -308, -1044, + 308, -1044, 996, -1044, -996, -1044, 991, -1044, -991, -1044, 958, + -1044, -958, -1044, -1460, -1044, 1460, -1044, 1522, -1044, -1522, -1044, + 1628, -1044, -1628, +}; diff --git a/src/mlkem-native/mlkem/src/native/x86_64/README.md b/src/mlkem-native/mlkem/src/native/x86_64/README.md new file mode 100644 index 0000000000..2073425c3b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/README.md @@ -0,0 +1,4 @@ +[//]: # (SPDX-License-Identifier: CC-BY-4.0) + +This directory contains the native x86_64 arithmetic backend for ML-KEM provided by the official [AVX2 +implementation](https://github.com/pq-crystals/kyber/tree/main/avx2) of the Kyber team. diff --git a/src/mlkem-native/mlkem/src/native/x86_64/meta.h b/src/mlkem-native/mlkem/src/native/x86_64/meta.h new file mode 100644 index 0000000000..073b233df0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/meta.h @@ -0,0 +1,324 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLK_NATIVE_X86_64_META_H +#define MLK_NATIVE_X86_64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLK_ARITH_BACKEND_X86_64_DEFAULT + +#define MLK_USE_NATIVE_NTT_CUSTOM_ORDER +#define MLK_USE_NATIVE_REJ_UNIFORM +#define MLK_USE_NATIVE_NTT +#define MLK_USE_NATIVE_INTT +#define MLK_USE_NATIVE_POLY_REDUCE +#define MLK_USE_NATIVE_POLY_TOMONT +#define MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED +#define MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE +#define MLK_USE_NATIVE_POLY_TOBYTES +#define MLK_USE_NATIVE_POLY_FROMBYTES +#define MLK_USE_NATIVE_POLY_COMPRESS_D4 +#define MLK_USE_NATIVE_POLY_COMPRESS_D5 +#define MLK_USE_NATIVE_POLY_COMPRESS_D10 +#define MLK_USE_NATIVE_POLY_COMPRESS_D11 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D4 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D5 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D10 +#define MLK_USE_NATIVE_POLY_DECOMPRESS_D11 + +#if !defined(__ASSEMBLER__) +#include "../../common.h" +#include "../api.h" +#include "src/arith_native_x86_64.h" +#include "src/compress_consts.h" + +static MLK_INLINE void mlk_poly_permute_bitrev_to_custom(int16_t data[MLKEM_N]) +{ + if (mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + mlk_nttunpack_avx2_asm(data); + } +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2) || len != MLKEM_N || + buflen % 12 != 0) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + return (int)mlk_rej_uniform_avx2_asm(r, buf, buflen, mlk_rej_uniform_table); +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_ntt_avx2_asm(data, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_invntt_avx2_asm(data, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_reduce_avx2_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_tomont_avx2_asm(data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2], + const int16_t y[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_mulcache_compute_avx2_asm(x, y, mlk_qdata); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N], + const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N], + const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3 */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4 +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N], + const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(r, a, b, b_cache); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4 */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES], + const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_ntttobytes_avx2_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_frombytes_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYBYTES]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_nttfrombytes_avx2_asm(r, a); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \ + MLKEM_K == 3) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d4_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d4_avx2_asm(r, a, mlk_compress_d4_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d10_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d10_avx2_asm(r, a, mlk_compress_d10_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d4_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d4_avx2_asm(r, a, mlk_decompress_d4_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d10_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d10_avx2_asm(r, a, mlk_decompress_d10_data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d5_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d5_avx2_asm(r, a, mlk_compress_d5_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_compress_d11_native( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], const int16_t a[MLKEM_N]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_compress_d11_avx2_asm(r, a, mlk_compress_d11_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d5_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d5_avx2_asm(r, a, mlk_decompress_d5_data); + return MLK_NATIVE_FUNC_SUCCESS; +} + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_poly_decompress_d11_native( + int16_t r[MLKEM_N], const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11]) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_X86_64_AVX2)) + { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_poly_decompress_d11_avx2_asm(r, a, mlk_decompress_d11_data); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ +#endif /* (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_X86_64_META_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h b/src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h new file mode 100644 index 0000000000..b4423a3981 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/arith_native_x86_64.h @@ -0,0 +1,327 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H +#define MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H + +#include "../../../common.h" + +#include +#include "compress_consts.h" +#include "consts.h" + +#define MLK_AVX2_REJ_UNIFORM_BUFLEN \ + (3 * 168) /* REJ_UNIFORM_NBLOCKS * SHAKE128_RATE */ + +#define mlk_rej_uniform_table MLK_NAMESPACE(rej_uniform_table) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_rej_uniform_table[4096]; + +#define mlk_rej_uniform_avx2_asm MLK_NAMESPACE(rej_uniform_avx2_asm) +MLK_MUST_CHECK_RETURN_VALUE MLK_SYSV_ABI +uint64_t mlk_rej_uniform_avx2_asm(int16_t *r, const uint8_t *buf, + unsigned buflen, const uint8_t *table) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_rej_uniform_avx2_asm.ml. */ +__contract__( + requires(buflen % 12 == 0) + requires(memory_no_alias(buf, buflen)) + requires(table == mlk_rej_uniform_table) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(return_value <= MLKEM_N) + ensures(array_bound(r, 0, (unsigned) return_value, 0, MLKEM_Q)) +); + +#define mlk_ntt_avx2_asm MLK_NAMESPACE(ntt_avx2_asm) +MLK_SYSV_ABI +void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_ntt_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(r, 0, MLKEM_N, 8192)) + requires(qdata == mlk_qdata) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(r, 0, MLKEM_N, 23595)) + /* check-magic: on */ +); + +#define mlk_invntt_avx2_asm MLK_NAMESPACE(invntt_avx2_asm) +MLK_SYSV_ABI +void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_intt_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(qdata == mlk_qdata) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* check-magic: off */ + ensures(array_abs_bound(r, 0, MLKEM_N, 26632)) + /* check-magic: on */ +); + +#define mlk_nttunpack_avx2_asm MLK_NAMESPACE(nttunpack_avx2_asm) +MLK_SYSV_ABI +void mlk_nttunpack_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_nttunpack_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + /* Output is a permutation of input: every output coefficient + * is some input coefficient */ + ensures(forall(i, 0, MLKEM_N, exists(j, 0, MLKEM_N, + r[i] == old(*(int16_t (*)[MLKEM_N])r)[j]))) +); + +#define mlk_reduce_avx2_asm MLK_NAMESPACE(reduce_avx2_asm) +MLK_SYSV_ABI +void mlk_reduce_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_reduce_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_mulcache_compute_avx2_asm \ + MLK_NAMESPACE(poly_mulcache_compute_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_mulcache_compute_avx2_asm(int16_t *out, const int16_t *in, + const int16_t *qdata) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/mlkem_poly_mulcache_compute_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(out, sizeof(int16_t) * (MLKEM_N / 2))) + requires(memory_no_alias(in, sizeof(int16_t) * MLKEM_N)) + requires(qdata == mlk_qdata) + assigns(memory_slice(out, sizeof(int16_t) * (MLKEM_N / 2))) + ensures(array_abs_bound(out, 0, MLKEM_N/2, MLKEM_Q)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 2 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 2 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 2 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 3 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 3 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 3 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm \ + MLK_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) +MLK_SYSV_ABI +void mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm( + int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b, sizeof(int16_t) * 4 * MLKEM_N)) + requires(memory_no_alias(b_cache, sizeof(int16_t) * 4 * (MLKEM_N / 2))) + requires(array_abs_bound(a, 0, 4 * MLKEM_N, MLKEM_UINT12_LIMIT + 1)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) +); + +#define mlk_ntttobytes_avx2_asm MLK_NAMESPACE(ntttobytes_avx2_asm) +MLK_SYSV_ABI +void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_ntttobytes_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYBYTES)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYBYTES)) +); + +#define mlk_nttfrombytes_avx2_asm MLK_NAMESPACE(nttfrombytes_avx2_asm) +MLK_SYSV_ABI +void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_nttfrombytes_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(a, MLKEM_POLYBYTES)) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT)) +); + +#define mlk_tomont_avx2_asm MLK_NAMESPACE(tomont_avx2_asm) +MLK_SYSV_ABI +void mlk_tomont_avx2_asm(int16_t *r) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_tomont_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) +); + +#define mlk_poly_compress_d4_avx2_asm MLK_NAMESPACE(poly_compress_d4_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d4_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D4], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d4_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D4)) +); + +#define mlk_poly_decompress_d4_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d4_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d4_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D4], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d4_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D4)) + requires(data == mlk_decompress_d4_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d10_avx2_asm MLK_NAMESPACE(poly_compress_d10_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d10_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D10], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d10_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d10_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D10)) +); + +#define mlk_poly_decompress_d10_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d10_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d10_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D10], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d10_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D10)) + requires(data == mlk_decompress_d10_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d5_avx2_asm MLK_NAMESPACE(poly_compress_d5_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d5_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D5], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d5_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d5_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D5)) +); + +#define mlk_poly_decompress_d5_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d5_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d5_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D5], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d5_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D5)) + requires(data == mlk_decompress_d5_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_compress_d11_avx2_asm MLK_NAMESPACE(poly_compress_d11_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_compress_d11_avx2_asm(uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_D11], + const int16_t *MLK_RESTRICT a, + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_compress_d11_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(memory_no_alias(a, sizeof(int16_t) * MLKEM_N)) + requires(array_bound(a, 0, MLKEM_N, 0, MLKEM_Q)) + requires(data == mlk_compress_d11_data) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_D11)) +); + +#define mlk_poly_decompress_d11_avx2_asm \ + MLK_NAMESPACE(poly_decompress_d11_avx2_asm) +MLK_SYSV_ABI +void mlk_poly_decompress_d11_avx2_asm( + int16_t *MLK_RESTRICT r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_D11], + const uint8_t *data) +/* This must be kept in sync with the HOL-Light specification in + * proofs/hol_light/x86_64/proofs/mlkem_poly_decompress_d11_avx2_asm.ml. + */ +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_D11)) + requires(data == mlk_decompress_d11_data) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_bound(r, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#endif /* !MLK_NATIVE_X86_64_SRC_ARITH_NATIVE_X86_64_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c new file mode 100644 index 0000000000..1645fca573 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) + +#include "compress_consts.h" + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || \ + MLKEM_K == 3) + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d4_data[32] = { + 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, + 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, /* permdidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d4_data[32] = { + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, /* shufbidx */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d10_data[32] = { + 0, 1, 2, 3, 4, 8, 9, 10, 11, 12, 255, + 255, 255, 255, 255, 255, 9, 10, 11, 12, 255, 255, + 255, 255, 255, 255, 0, 1, 2, 3, 4, 8, /* shufbidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d10_data[32] = { + 0, 1, 1, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 9, + 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, /* shufbidx */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) + +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d5_data[32] = { + 0, 1, 2, 3, 4, 255, 255, 255, 255, 255, 8, + 9, 10, 11, 12, 255, 9, 10, 11, 12, 255, 0, + 1, 2, 3, 4, 255, 255, 255, 255, 255, 8, /* shufbidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* shufbidx[0:32], mask[32:64], shift[64:96] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d5_data[96] = { + 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, + 5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 9, /* shufbidx */ + 31, 0, 224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, 31, 0, + 224, 3, 124, 0, 128, 15, 240, 1, 62, 0, 192, 7, 248, 0, /* mask */ + 0, 4, 32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, 0, 4, + 32, 0, 0, 1, 8, 0, 64, 0, 0, 2, 16, 0, 128, 0, /* shift */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +/* srlvqidx[0:32], shufbidx[32:64] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_compress_d11_data[64] = + { + 10, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, + 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, + 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 255, 255, 255, 255, 255, 5, 6, 7, 8, 9, 10, + 255, 255, 255, 255, 0, 0, 1, 2, 3, 4, /* shufbidx */ +}; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* shufbidx[0:32], srlvdidx[32:64], srlvqidx[64:96], shift[96:128] */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_decompress_d11_data[128] = { + 0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10, + 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 9, 10, 11, 12, 12, 13, /* shufbidx */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* srlvdidx */ + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, /* srlvqidx */ + 32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0, + 32, 0, 4, 0, 1, 0, 32, 0, 8, 0, 1, 0, 32, 0, 4, 0, /* shift */ +}; +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_CONFIG_MULTILEVEL_NO_SHARED && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +MLK_EMPTY_CU(avx2_compress_consts) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API)) */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h new file mode 100644 index 0000000000..79182b0052 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/compress_consts.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H +#define MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H + +#include "../../../common.h" + +#ifndef __ASSEMBLER__ + +#define mlk_compress_d4_data MLK_NAMESPACE(compress_d4_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d4_data[32]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d4_data MLK_NAMESPACE(decompress_d4_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d4_data[32]; +#endif + +#define mlk_compress_d10_data MLK_NAMESPACE(compress_d10_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d10_data[32]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d10_data MLK_NAMESPACE(decompress_d10_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d10_data[32]; +#endif + +#define mlk_compress_d5_data MLK_NAMESPACE(compress_d5_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d5_data[32]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d5_data MLK_NAMESPACE(decompress_d5_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d5_data[96]; +#endif + +#define mlk_compress_d11_data MLK_NAMESPACE(compress_d11_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_compress_d11_data[64]; + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_decompress_d11_data MLK_NAMESPACE(decompress_d11_data) +MLK_INTERNAL_DATA_DECLARATION const uint8_t mlk_decompress_d11_data[128]; +#endif + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_NATIVE_X86_64_SRC_COMPRESS_CONSTS_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/consts.c b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.c new file mode 100644 index 0000000000..d0acfe0f9c --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "consts.h" + +/* + * Table of zeta values used in the AVX2 NTTs + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const int16_t mlk_qdata[624] = { + 3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 3854, + 3340, 2826, 2312, 1798, 1284, 770, 256, 7, 0, + 6, 0, 5, 0, 4, 0, 3, 0, 2, + 0, 1, 0, 0, 0, 31498, 31498, 31498, 31498, + -758, -758, -758, -758, 0, 0, 0, 0, 0, + 0, 0, 0, 14745, 14745, 14745, 14745, 14745, 14745, + 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, + 14745, -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, 13525, + 13525, 13525, 13525, 13525, 13525, 13525, 13525, -12402, -12402, + -12402, -12402, -12402, -12402, -12402, -12402, 1493, 1493, 1493, + 1493, 1493, 1493, 1493, 1493, 1422, 1422, 1422, 1422, + 1422, 1422, 1422, 1422, -20907, -20907, -20907, -20907, 27758, + 27758, 27758, 27758, -3799, -3799, -3799, -3799, -15690, -15690, + -15690, -15690, -171, -171, -171, -171, 622, 622, 622, + 622, 1577, 1577, 1577, 1577, 182, 182, 182, 182, + -5827, -5827, 17363, 17363, -26360, -26360, -29057, -29057, 5571, + 5571, -1102, -1102, 21438, 21438, -26242, -26242, 573, 573, + -1325, -1325, 264, 264, 383, 383, -829, -829, 1458, + 1458, -1602, -1602, -130, -130, -5689, -6516, 1496, 30967, + -23565, 20179, 20710, 25080, -12796, 26616, 16064, -12442, 9134, + -650, -25986, 27837, 1223, 652, -552, 1015, -1293, 1491, + -282, -1544, 516, -8, -320, -666, -1618, -1162, 126, + 1469, -335, -11477, -32227, 20494, -27738, 945, -14883, 6182, + 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, -1103, + 555, -1251, 1550, 422, 177, -291, 1574, -246, 1159, + -777, -602, -1590, -872, 418, -156, 11182, 13387, -14233, + -21655, 13131, -4587, 23092, 5493, -32502, 30317, -18741, 12639, + 20100, 18525, 19529, -12619, 430, 843, 871, 105, 587, + -235, -460, 1653, 778, -147, 1483, 1119, 644, 349, + 329, -75, 787, 787, 787, 787, 787, 787, 787, + 787, 787, 787, 787, 787, 787, 787, 787, 787, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, 28191, 28191, + 28191, 28191, 28191, 28191, 28191, 28191, -16694, -16694, -16694, + -16694, -16694, -16694, -16694, -16694, 287, 287, 287, 287, + 287, 287, 287, 287, 202, 202, 202, 202, 202, + 202, 202, 202, 10690, 10690, 10690, 10690, 1358, 1358, + 1358, 1358, -11202, -11202, -11202, -11202, 31164, 31164, 31164, + 31164, 962, 962, 962, 962, -1202, -1202, -1202, -1202, + -1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468, -28073, + -28073, 24313, 24313, -10532, -10532, 8800, 8800, 18426, 18426, + 8859, 8859, 26675, 26675, -16163, -16163, -681, -681, 1017, + 1017, 732, 732, 608, 608, -1542, -1542, 411, 411, + -205, -205, -1571, -1571, 19883, -28250, -15887, -8898, -28309, + 9075, -30199, 18249, 13426, 14017, -29156, -12757, 16832, 4311, + -24155, -17915, -853, -90, -271, 830, 107, -1421, -247, + -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, + -31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335, + -7934, -22502, 10906, 31636, 28644, 23998, -17422, 817, 603, + 1322, -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, + -870, -108, 996, 958, 1522, 20297, 2146, 15355, -32384, + -6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860, + -20257, 7756, 23132, 1097, 610, -1285, 384, -136, -1335, + 220, -1659, -1530, 794, -854, 478, -308, 991, -1460, + 1628, -1103, 555, -1251, 1550, 422, 177, -291, 1574, + -246, 1159, -777, -602, -1590, -872, 418, -156, 430, + 843, 871, 105, 587, -235, -460, 1653, 778, -147, + 1483, 1119, 644, 349, 329, -75, 817, 603, 1322, + -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, -870, + -108, 996, 958, 1522, 1097, 610, -1285, 384, -136, + -1335, 220, -1659, -1530, 794, -854, 478, -308, 991, + -1460, 1628, -335, -11477, -32227, 20494, -27738, 945, -14883, + 6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, + 11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493, -32502, + 30317, -18741, 12639, 20100, 18525, 19529, -12619, -31183, 25435, + -7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502, + 10906, 31636, 28644, 23998, -17422, 20297, 2146, 15355, -32384, + -6280, -14903, -11044, 14469, -21498, -20198, 23210, -17442, -23860, + -20257, 7756, 23132, +}; + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(avx2_consts) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/consts.h b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.h new file mode 100644 index 0000000000..591529d471 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/consts.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#ifndef MLK_NATIVE_X86_64_SRC_CONSTS_H +#define MLK_NATIVE_X86_64_SRC_CONSTS_H +#include "../../../common.h" +#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXB 0 +#define MLK_AVX2_BACKEND_DATA_OFFSET_REVIDXD 16 +#define MLK_AVX2_BACKEND_DATA_OFFSET_ZETAS_EXP 32 +#define MLK_AVX2_BACKEND_DATA_OFFSET_MULCACHE_TWIDDLES 496 + +#ifndef __ASSEMBLER__ +#define mlk_qdata MLK_NAMESPACE(qdata) +MLK_INTERNAL_DATA_DECLARATION const int16_t mlk_qdata[624]; +#endif + +#endif /* !MLK_NATIVE_X86_64_SRC_CONSTS_H */ diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S new file mode 100644 index 0000000000..ca76612184 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_intt_avx2_asm.S @@ -0,0 +1,743 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [AVX2_NTT] + * Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography. + * Gregor Seiler + * https://eprint.iacr.org/2018/039 + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * The core ideas behind the implementation are described in @[AVX2_NTT]. + * + * Changes: + * - Different placement of modular reductions to simplify + * reasoning of non-overflow + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) +/*yaml + Name: invntt_avx2_asm + Description: x86_64 AVX2 inverse NTT + Signature: void mlk_invntt_avx2_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_intt_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(invntt_avx2_asm) +MLK_ASM_FN_SYMBOL(invntt_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x5a105a1, %eax # imm = 0x5A105A1 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x60(%rdi), %ymm7 + vpmullw %ymm2, %ymm4, %ymm12 + vpmulhw %ymm3, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm2, %ymm6, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmullw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmullw %ymm2, %ymm7, %ymm12 + vpmulhw %ymm3, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xe0(%rdi), %ymm11 + vpmullw %ymm2, %ymm8, %ymm12 + vpmulhw %ymm3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmullw %ymm2, %ymm10, %ymm12 + vpmulhw %ymm3, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpmullw %ymm2, %ymm9, %ymm12 + vpmulhw %ymm3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm2, %ymm11, %ymm12 + vpmulhw %ymm3, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm11, %ymm11 + vpermq $0x4e, 0x3a0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1] + vpermq $0x4e, 0x360(%rsi), %ymm1 # ymm1 = mem[2,3,0,1] + vpermq $0x4e, 0x3c0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x380(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm12 + vpshufb %ymm12, %ymm15, %ymm15 + vpshufb %ymm12, %ymm1, %ymm1 + vpshufb %ymm12, %ymm2, %ymm2 + vpshufb %ymm12, %ymm3, %ymm3 + vpsubw %ymm4, %ymm6, %ymm12 + vpaddw %ymm6, %ymm4, %ymm4 + vpsubw %ymm5, %ymm7, %ymm13 + vpmullw %ymm15, %ymm12, %ymm6 + vpaddw %ymm7, %ymm5, %ymm5 + vpsubw %ymm8, %ymm10, %ymm14 + vpmullw %ymm15, %ymm13, %ymm7 + vpaddw %ymm10, %ymm8, %ymm8 + vpsubw %ymm9, %ymm11, %ymm15 + vpmullw %ymm1, %ymm14, %ymm10 + vpaddw %ymm11, %ymm9, %ymm9 + vpmullw %ymm1, %ymm15, %ymm11 + vpmulhw %ymm2, %ymm12, %ymm12 + vpmulhw %ymm2, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm6, %ymm12, %ymm6 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpermq $0x4e, 0x320(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x340(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm1 + vpshufb %ymm1, %ymm2, %ymm2 + vpshufb %ymm1, %ymm3, %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpslld $0x10, %ymm11, %ymm8 + vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15] + vpsrld $0x10, %ymm10, %ymm10 + vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15] + vmovdqa 0x20(%rsi), %ymm12 + vpermd 0x2e0(%rsi), %ymm12, %ymm2 + vpermd 0x300(%rsi), %ymm12, %ymm10 + vpsubw %ymm3, %ymm5, %ymm12 + vpaddw %ymm5, %ymm3, %ymm3 + vpsubw %ymm4, %ymm7, %ymm13 + vpmullw %ymm2, %ymm12, %ymm5 + vpaddw %ymm7, %ymm4, %ymm4 + vpsubw %ymm6, %ymm9, %ymm14 + vpmullw %ymm2, %ymm13, %ymm7 + vpaddw %ymm9, %ymm6, %ymm6 + vpsubw %ymm8, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm9 + vpaddw %ymm11, %ymm8, %ymm8 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm10, %ymm12, %ymm12 + vpmulhw %ymm10, %ymm13, %ymm13 + vpmulhw %ymm10, %ymm14, %ymm14 + vpmulhw %ymm10, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm5, %ymm12, %ymm5 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm9, %ymm14, %ymm9 + vpsubw %ymm11, %ymm15, %ymm11 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7] + vpsrlq $0x20, %ymm9, %ymm9 + vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7] + vpermq $0x1b, 0x2a0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0] + vpermq $0x1b, 0x2c0(%rsi), %ymm9 # ymm9 = mem[3,2,1,0] + vpsubw %ymm10, %ymm4, %ymm12 + vpaddw %ymm4, %ymm10, %ymm10 + vpsubw %ymm3, %ymm8, %ymm13 + vpmullw %ymm2, %ymm12, %ymm4 + vpaddw %ymm8, %ymm3, %ymm3 + vpsubw %ymm6, %ymm7, %ymm14 + vpmullw %ymm2, %ymm13, %ymm8 + vpaddw %ymm7, %ymm6, %ymm6 + vpsubw %ymm5, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm7 + vpaddw %ymm11, %ymm5, %ymm5 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm9, %ymm12, %ymm12 + vpmulhw %ymm9, %ymm13, %ymm13 + vpmulhw %ymm9, %ymm14, %ymm14 + vpmulhw %ymm9, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm4, %ymm12, %ymm4 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm7, %ymm14, %ymm7 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm10, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3] + vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3] + vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3] + vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3] + vpermq $0x4e, 0x260(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x280(%rsi), %ymm7 # ymm7 = mem[2,3,0,1] + vpsubw %ymm9, %ymm3, %ymm12 + vpaddw %ymm3, %ymm9, %ymm9 + vpsubw %ymm10, %ymm5, %ymm13 + vpmullw %ymm2, %ymm12, %ymm3 + vpaddw %ymm5, %ymm10, %ymm10 + vpsubw %ymm6, %ymm8, %ymm14 + vpmullw %ymm2, %ymm13, %ymm5 + vpaddw %ymm8, %ymm6, %ymm6 + vpsubw %ymm4, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm8 + vpaddw %ymm11, %ymm4, %ymm4 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm7, %ymm12, %ymm12 + vpmulhw %ymm7, %ymm13, %ymm13 + vpmulhw %ymm7, %ymm14, %ymm14 + vpmulhw %ymm7, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm3, %ymm12, %ymm3 + vpsubw %ymm5, %ymm13, %ymm5 + vpsubw %ymm8, %ymm14, %ymm8 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1] + vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3] + vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3] + vmovdqa 0x220(%rsi), %ymm2 + vmovdqa 0x240(%rsi), %ymm8 + vpsubw %ymm7, %ymm10, %ymm12 + vpaddw %ymm10, %ymm7, %ymm7 + vpsubw %ymm9, %ymm4, %ymm13 + vpmullw %ymm2, %ymm12, %ymm10 + vpaddw %ymm4, %ymm9, %ymm9 + vpsubw %ymm6, %ymm5, %ymm14 + vpmullw %ymm2, %ymm13, %ymm4 + vpaddw %ymm5, %ymm6, %ymm6 + vpsubw %ymm3, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm5 + vpaddw %ymm11, %ymm3, %ymm3 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm8, %ymm12, %ymm12 + vpmulhw %ymm8, %ymm13, %ymm13 + vpmulhw %ymm8, %ymm14, %ymm14 + vpmulhw %ymm8, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm10, %ymm12, %ymm10 + vpsubw %ymm4, %ymm13, %ymm4 + vpsubw %ymm5, %ymm14, %ymm5 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa %ymm6, 0x40(%rdi) + vmovdqa %ymm3, 0x60(%rdi) + vmovdqa %ymm10, 0x80(%rdi) + vmovdqa %ymm4, 0xa0(%rdi) + vmovdqa %ymm5, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + movl $0xd8a1d8a1, %eax # imm = 0xD8A1D8A1 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x5a105a1, %eax # imm = 0x5A105A1 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x160(%rdi), %ymm7 + vpmullw %ymm2, %ymm4, %ymm12 + vpmulhw %ymm3, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm2, %ymm6, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmullw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmullw %ymm2, %ymm7, %ymm12 + vpmulhw %ymm3, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1e0(%rdi), %ymm11 + vpmullw %ymm2, %ymm8, %ymm12 + vpmulhw %ymm3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmullw %ymm2, %ymm10, %ymm12 + vpmulhw %ymm3, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpmullw %ymm2, %ymm9, %ymm12 + vpmulhw %ymm3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm2, %ymm11, %ymm12 + vpmulhw %ymm3, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm11, %ymm11 + vpermq $0x4e, 0x1e0(%rsi), %ymm15 # ymm15 = mem[2,3,0,1] + vpermq $0x4e, 0x1a0(%rsi), %ymm1 # ymm1 = mem[2,3,0,1] + vpermq $0x4e, 0x200(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x1c0(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm12 + vpshufb %ymm12, %ymm15, %ymm15 + vpshufb %ymm12, %ymm1, %ymm1 + vpshufb %ymm12, %ymm2, %ymm2 + vpshufb %ymm12, %ymm3, %ymm3 + vpsubw %ymm4, %ymm6, %ymm12 + vpaddw %ymm6, %ymm4, %ymm4 + vpsubw %ymm5, %ymm7, %ymm13 + vpmullw %ymm15, %ymm12, %ymm6 + vpaddw %ymm7, %ymm5, %ymm5 + vpsubw %ymm8, %ymm10, %ymm14 + vpmullw %ymm15, %ymm13, %ymm7 + vpaddw %ymm10, %ymm8, %ymm8 + vpsubw %ymm9, %ymm11, %ymm15 + vpmullw %ymm1, %ymm14, %ymm10 + vpaddw %ymm11, %ymm9, %ymm9 + vpmullw %ymm1, %ymm15, %ymm11 + vpmulhw %ymm2, %ymm12, %ymm12 + vpmulhw %ymm2, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm6, %ymm12, %ymm6 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpermq $0x4e, 0x160(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0x180(%rsi), %ymm3 # ymm3 = mem[2,3,0,1] + vmovdqa (%rsi), %ymm1 + vpshufb %ymm1, %ymm2, %ymm2 + vpshufb %ymm1, %ymm3, %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpslld $0x10, %ymm11, %ymm8 + vpblendw $0xaa, %ymm8, %ymm10, %ymm8 # ymm8 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4],ymm8[5],ymm10[6],ymm8[7],ymm10[8],ymm8[9],ymm10[10],ymm8[11],ymm10[12],ymm8[13],ymm10[14],ymm8[15] + vpsrld $0x10, %ymm10, %ymm10 + vpblendw $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7],ymm10[8],ymm11[9],ymm10[10],ymm11[11],ymm10[12],ymm11[13],ymm10[14],ymm11[15] + vmovdqa 0x20(%rsi), %ymm12 + vpermd 0x120(%rsi), %ymm12, %ymm2 + vpermd 0x140(%rsi), %ymm12, %ymm10 + vpsubw %ymm3, %ymm5, %ymm12 + vpaddw %ymm5, %ymm3, %ymm3 + vpsubw %ymm4, %ymm7, %ymm13 + vpmullw %ymm2, %ymm12, %ymm5 + vpaddw %ymm7, %ymm4, %ymm4 + vpsubw %ymm6, %ymm9, %ymm14 + vpmullw %ymm2, %ymm13, %ymm7 + vpaddw %ymm9, %ymm6, %ymm6 + vpsubw %ymm8, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm9 + vpaddw %ymm11, %ymm8, %ymm8 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm10, %ymm12, %ymm12 + vpmulhw %ymm10, %ymm13, %ymm13 + vpmulhw %ymm10, %ymm14, %ymm14 + vpmulhw %ymm10, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm5, %ymm12, %ymm5 + vpsubw %ymm7, %ymm13, %ymm7 + vpsubw %ymm9, %ymm14, %ymm9 + vpsubw %ymm11, %ymm15, %ymm11 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vmovsldup %ymm4, %ymm10 # ymm10 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm8, %ymm3 # ymm3 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm8, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vmovsldup %ymm11, %ymm5 # ymm5 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7] + vpsrlq $0x20, %ymm9, %ymm9 + vpblendd $0xaa, %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[0],ymm11[1],ymm9[2],ymm11[3],ymm9[4],ymm11[5],ymm9[6],ymm11[7] + vpermq $0x1b, 0xe0(%rsi), %ymm2 # ymm2 = mem[3,2,1,0] + vpermq $0x1b, 0x100(%rsi), %ymm9 # ymm9 = mem[3,2,1,0] + vpsubw %ymm10, %ymm4, %ymm12 + vpaddw %ymm4, %ymm10, %ymm10 + vpsubw %ymm3, %ymm8, %ymm13 + vpmullw %ymm2, %ymm12, %ymm4 + vpaddw %ymm8, %ymm3, %ymm3 + vpsubw %ymm6, %ymm7, %ymm14 + vpmullw %ymm2, %ymm13, %ymm8 + vpaddw %ymm7, %ymm6, %ymm6 + vpsubw %ymm5, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm7 + vpaddw %ymm11, %ymm5, %ymm5 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm9, %ymm12, %ymm12 + vpmulhw %ymm9, %ymm13, %ymm13 + vpmulhw %ymm9, %ymm14, %ymm14 + vpmulhw %ymm9, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm4, %ymm12, %ymm4 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm7, %ymm14, %ymm7 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm10, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm10, %ymm10 + vpunpcklqdq %ymm3, %ymm10, %ymm9 # ymm9 = ymm10[0],ymm3[0],ymm10[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[1],ymm3[1],ymm10[3],ymm3[3] + vpunpcklqdq %ymm5, %ymm6, %ymm10 # ymm10 = ymm6[0],ymm5[0],ymm6[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3] + vpunpcklqdq %ymm8, %ymm4, %ymm6 # ymm6 = ymm4[0],ymm8[0],ymm4[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[1],ymm8[1],ymm4[3],ymm8[3] + vpunpcklqdq %ymm11, %ymm7, %ymm4 # ymm4 = ymm7[0],ymm11[0],ymm7[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[1],ymm11[1],ymm7[3],ymm11[3] + vpermq $0x4e, 0xa0(%rsi), %ymm2 # ymm2 = mem[2,3,0,1] + vpermq $0x4e, 0xc0(%rsi), %ymm7 # ymm7 = mem[2,3,0,1] + vpsubw %ymm9, %ymm3, %ymm12 + vpaddw %ymm3, %ymm9, %ymm9 + vpsubw %ymm10, %ymm5, %ymm13 + vpmullw %ymm2, %ymm12, %ymm3 + vpaddw %ymm5, %ymm10, %ymm10 + vpsubw %ymm6, %ymm8, %ymm14 + vpmullw %ymm2, %ymm13, %ymm5 + vpaddw %ymm8, %ymm6, %ymm6 + vpsubw %ymm4, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm8 + vpaddw %ymm11, %ymm4, %ymm4 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm7, %ymm12, %ymm12 + vpmulhw %ymm7, %ymm13, %ymm13 + vpmulhw %ymm7, %ymm14, %ymm14 + vpmulhw %ymm7, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm3, %ymm12, %ymm3 + vpsubw %ymm5, %ymm13, %ymm5 + vpsubw %ymm8, %ymm14, %ymm8 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vperm2i128 $0x20, %ymm10, %ymm9, %ymm7 # ymm7 = ymm9[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm4, %ymm6, %ymm9 # ymm9 = ymm6[0,1],ymm4[0,1] + vperm2i128 $0x31, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[2,3],ymm4[2,3] + vperm2i128 $0x20, %ymm5, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm5[0,1] + vperm2i128 $0x31, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[2,3],ymm5[2,3] + vperm2i128 $0x20, %ymm11, %ymm8, %ymm3 # ymm3 = ymm8[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm8, %ymm11 # ymm11 = ymm8[2,3],ymm11[2,3] + vmovdqa 0x60(%rsi), %ymm2 + vmovdqa 0x80(%rsi), %ymm8 + vpsubw %ymm7, %ymm10, %ymm12 + vpaddw %ymm10, %ymm7, %ymm7 + vpsubw %ymm9, %ymm4, %ymm13 + vpmullw %ymm2, %ymm12, %ymm10 + vpaddw %ymm4, %ymm9, %ymm9 + vpsubw %ymm6, %ymm5, %ymm14 + vpmullw %ymm2, %ymm13, %ymm4 + vpaddw %ymm5, %ymm6, %ymm6 + vpsubw %ymm3, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm5 + vpaddw %ymm11, %ymm3, %ymm3 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm8, %ymm12, %ymm12 + vpmulhw %ymm8, %ymm13, %ymm13 + vpmulhw %ymm8, %ymm14, %ymm14 + vpmulhw %ymm8, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm10, %ymm12, %ymm10 + vpsubw %ymm4, %ymm13, %ymm4 + vpsubw %ymm5, %ymm14, %ymm5 + vpsubw %ymm11, %ymm15, %ymm11 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm6, 0x140(%rdi) + vmovdqa %ymm3, 0x160(%rdi) + vmovdqa %ymm10, 0x180(%rdi) + vmovdqa %ymm4, 0x1a0(%rdi) + vmovdqa %ymm5, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + vmovdqa (%rdi), %ymm4 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x120(%rdi), %ymm9 + vpbroadcastq 0x40(%rsi), %ymm2 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x140(%rdi), %ymm10 + vmovdqa 0x60(%rdi), %ymm7 + vmovdqa 0x160(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vmovdqa %ymm4, (%rdi) + vmovdqa %ymm5, 0x20(%rdi) + vmovdqa %ymm6, 0x40(%rdi) + vmovdqa %ymm7, 0x60(%rdi) + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm11, 0x160(%rdi) + vmovdqa 0x80(%rdi), %ymm4 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm5 + vmovdqa 0x1a0(%rdi), %ymm9 + vpbroadcastq 0x40(%rsi), %ymm2 + vmovdqa 0xc0(%rdi), %ymm6 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm7 + vmovdqa 0x1e0(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm3 + vpsubw %ymm4, %ymm8, %ymm12 + vpaddw %ymm8, %ymm4, %ymm4 + vpsubw %ymm5, %ymm9, %ymm13 + vpmullw %ymm2, %ymm12, %ymm8 + vpaddw %ymm9, %ymm5, %ymm5 + vpsubw %ymm6, %ymm10, %ymm14 + vpmullw %ymm2, %ymm13, %ymm9 + vpaddw %ymm10, %ymm6, %ymm6 + vpsubw %ymm7, %ymm11, %ymm15 + vpmullw %ymm2, %ymm14, %ymm10 + vpaddw %ymm11, %ymm7, %ymm7 + vpmullw %ymm2, %ymm15, %ymm11 + vpmulhw %ymm3, %ymm12, %ymm12 + vpmulhw %ymm3, %ymm13, %ymm13 + vpmulhw %ymm3, %ymm14, %ymm14 + vpmulhw %ymm3, %ymm15, %ymm15 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm8, %ymm12, %ymm8 + vpsubw %ymm9, %ymm13, %ymm9 + vpsubw %ymm10, %ymm14, %ymm10 + vpsubw %ymm11, %ymm15, %ymm11 + vmovdqa %ymm4, 0x80(%rdi) + vmovdqa %ymm5, 0xa0(%rdi) + vmovdqa %ymm6, 0xc0(%rdi) + vmovdqa %ymm7, 0xe0(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa %ymm10, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(invntt_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S new file mode 100644 index 0000000000..4e7894a411 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntt_avx2_asm.S @@ -0,0 +1,661 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [AVX2_NTT] + * Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography. + * Gregor Seiler + * https://eprint.iacr.org/2018/039 + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * The core ideas behind the implementation are described in @[AVX2_NTT]. + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: ntt_avx2_asm + Description: x86_64 AVX2 forward NTT + Signature: void mlk_ntt_avx2_asm(int16_t *r, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_ntt_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntt_avx2_asm) +MLK_ASM_FN_SYMBOL(ntt_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpbroadcastq 0x40(%rsi), %ymm15 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm9 + vmovdqa 0x140(%rdi), %ymm10 + vmovdqa 0x160(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm3, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm5, 0x40(%rdi) + vmovdqa %ymm6, 0x60(%rdi) + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm11, 0x160(%rdi) + vpbroadcastq 0x40(%rsi), %ymm15 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vpbroadcastq 0x48(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa 0x80(%rdi), %ymm4 + vmovdqa 0xa0(%rdi), %ymm5 + vmovdqa 0xc0(%rdi), %ymm6 + vmovdqa 0xe0(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm3, 0x80(%rdi) + vmovdqa %ymm4, 0xa0(%rdi) + vmovdqa %ymm5, 0xc0(%rdi) + vmovdqa %ymm6, 0xe0(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa %ymm10, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + vmovdqa 0x60(%rsi), %ymm15 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm11 + vmovdqa 0x80(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3] + vmovdqa 0xa0(%rsi), %ymm15 + vmovdqa 0xc0(%rsi), %ymm2 + vpmullw %ymm15, %ymm7, %ymm12 + vpmullw %ymm15, %ymm10, %ymm13 + vpmullw %ymm15, %ymm5, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm7, %ymm6, %ymm4 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm10, %ymm8, %ymm6 + vpsubw %ymm10, %ymm8, %ymm10 + vpaddw %ymm5, %ymm3, %ymm8 + vpsubw %ymm5, %ymm3, %ymm5 + vpaddw %ymm11, %ymm9, %ymm3 + vpsubw %ymm11, %ymm9, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm10, %ymm10 + vpsubw %ymm14, %ymm8, %ymm8 + vpaddw %ymm14, %ymm5, %ymm5 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3] + vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3] + vmovdqa 0xe0(%rsi), %ymm15 + vmovdqa 0x100(%rsi), %ymm2 + vpmullw %ymm15, %ymm9, %ymm12 + vpmullw %ymm15, %ymm5, %ymm13 + vpmullw %ymm15, %ymm8, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm11, %ymm11 + vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2] + vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3] + vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm9, %ymm3, %ymm6 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm5, %ymm7, %ymm3 + vpsubw %ymm5, %ymm7, %ymm5 + vpaddw %ymm8, %ymm4, %ymm7 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm11, %ymm10, %ymm4 + vpsubw %ymm11, %ymm10, %ymm11 + vpsubw %ymm12, %ymm6, %ymm6 + vpaddw %ymm12, %ymm9, %ymm9 + vpsubw %ymm13, %ymm3, %ymm3 + vpaddw %ymm13, %ymm5, %ymm5 + vpsubw %ymm14, %ymm7, %ymm7 + vpaddw %ymm14, %ymm8, %ymm8 + vpsubw %ymm15, %ymm4, %ymm4 + vpaddw %ymm15, %ymm11, %ymm11 + vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7] + vpsrlq $0x20, %ymm4, %ymm4 + vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7] + vmovdqa 0x120(%rsi), %ymm15 + vmovdqa 0x140(%rsi), %ymm2 + vpmullw %ymm15, %ymm10, %ymm12 + vpmullw %ymm15, %ymm8, %ymm13 + vpmullw %ymm15, %ymm7, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7] + vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm3 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm8, %ymm9, %ymm4 + vpsubw %ymm8, %ymm9, %ymm8 + vpaddw %ymm7, %ymm6, %ymm9 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm11, %ymm5, %ymm6 + vpsubw %ymm11, %ymm5, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm8, %ymm8 + vpsubw %ymm14, %ymm9, %ymm9 + vpaddw %ymm14, %ymm7, %ymm7 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vpslld $0x10, %ymm7, %ymm5 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15] + vpslld $0x10, %ymm11, %ymm9 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa 0x160(%rsi), %ymm15 + vmovdqa 0x180(%rsi), %ymm2 + vpmullw %ymm15, %ymm5, %ymm12 + vpmullw %ymm15, %ymm7, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpslld $0x10, %ymm10, %ymm6 + vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15] + vpsrld $0x10, %ymm3, %ymm3 + vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15] + vpslld $0x10, %ymm8, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm5, %ymm6, %ymm4 + vpsubw %ymm5, %ymm6, %ymm5 + vpaddw %ymm7, %ymm10, %ymm6 + vpsubw %ymm7, %ymm10, %ymm7 + vpaddw %ymm9, %ymm3, %ymm10 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm11, %ymm8, %ymm3 + vpsubw %ymm11, %ymm8, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm7, %ymm7 + vpsubw %ymm14, %ymm10, %ymm10 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa 0x1a0(%rsi), %ymm14 + vmovdqa 0x1e0(%rsi), %ymm15 + vmovdqa 0x1c0(%rsi), %ymm8 + vmovdqa 0x200(%rsi), %ymm2 + vpmullw %ymm14, %ymm10, %ymm12 + vpmullw %ymm14, %ymm3, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm8, %ymm10, %ymm10 + vpmulhw %ymm8, %ymm3, %ymm3 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm8 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm3, %ymm6, %ymm4 + vpsubw %ymm3, %ymm6, %ymm3 + vpaddw %ymm9, %ymm5, %ymm6 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm11, %ymm7, %ymm5 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm8, %ymm8 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm3, %ymm3 + vpsubw %ymm14, %ymm6, %ymm6 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm5, %ymm5 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm8, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm10, 0x40(%rdi) + vmovdqa %ymm3, 0x60(%rdi) + vmovdqa %ymm6, 0x80(%rdi) + vmovdqa %ymm5, 0xa0(%rdi) + vmovdqa %ymm9, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + vmovdqa 0x220(%rsi), %ymm15 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vmovdqa 0x240(%rsi), %ymm2 + vpmullw %ymm15, %ymm8, %ymm12 + vpmullw %ymm15, %ymm9, %ymm13 + vpmullw %ymm15, %ymm10, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x160(%rdi), %ymm7 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm8, %ymm4, %ymm3 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm9, %ymm5, %ymm4 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm10, %ymm6, %ymm5 + vpsubw %ymm10, %ymm6, %ymm10 + vpaddw %ymm11, %ymm7, %ymm6 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm9, %ymm9 + vpsubw %ymm14, %ymm5, %ymm5 + vpaddw %ymm14, %ymm10, %ymm10 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm10, %ymm5, %ymm7 # ymm7 = ymm5[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm5, %ymm10 # ymm10 = ymm5[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[2,3],ymm11[2,3] + vmovdqa 0x260(%rsi), %ymm15 + vmovdqa 0x280(%rsi), %ymm2 + vpmullw %ymm15, %ymm7, %ymm12 + vpmullw %ymm15, %ymm10, %ymm13 + vpmullw %ymm15, %ymm5, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm11, %ymm11 + vperm2i128 $0x20, %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm7, %ymm6, %ymm4 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm10, %ymm8, %ymm6 + vpsubw %ymm10, %ymm8, %ymm10 + vpaddw %ymm5, %ymm3, %ymm8 + vpsubw %ymm5, %ymm3, %ymm5 + vpaddw %ymm11, %ymm9, %ymm3 + vpsubw %ymm11, %ymm9, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm10, %ymm10 + vpsubw %ymm14, %ymm8, %ymm8 + vpaddw %ymm14, %ymm5, %ymm5 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vpunpcklqdq %ymm5, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm5[0],ymm8[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm8, %ymm5 # ymm5 = ymm8[1],ymm5[1],ymm8[3],ymm5[3] + vpunpcklqdq %ymm11, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm11[0],ymm3[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm3, %ymm11 # ymm11 = ymm3[1],ymm11[1],ymm3[3],ymm11[3] + vmovdqa 0x2a0(%rsi), %ymm15 + vmovdqa 0x2c0(%rsi), %ymm2 + vpmullw %ymm15, %ymm9, %ymm12 + vpmullw %ymm15, %ymm5, %ymm13 + vpmullw %ymm15, %ymm8, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm11, %ymm11 + vpunpcklqdq %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm7[0],ymm4[2],ymm7[2] + vpunpckhqdq %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[1],ymm7[1],ymm4[3],ymm7[3] + vpunpcklqdq %ymm10, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm10[0],ymm6[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[1],ymm10[1],ymm6[3],ymm10[3] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm9, %ymm3, %ymm6 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm5, %ymm7, %ymm3 + vpsubw %ymm5, %ymm7, %ymm5 + vpaddw %ymm8, %ymm4, %ymm7 + vpsubw %ymm8, %ymm4, %ymm8 + vpaddw %ymm11, %ymm10, %ymm4 + vpsubw %ymm11, %ymm10, %ymm11 + vpsubw %ymm12, %ymm6, %ymm6 + vpaddw %ymm12, %ymm9, %ymm9 + vpsubw %ymm13, %ymm3, %ymm3 + vpaddw %ymm13, %ymm5, %ymm5 + vpsubw %ymm14, %ymm7, %ymm7 + vpaddw %ymm14, %ymm8, %ymm8 + vpsubw %ymm15, %ymm4, %ymm4 + vpaddw %ymm15, %ymm11, %ymm11 + vmovsldup %ymm8, %ymm10 # ymm10 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm10, %ymm7, %ymm10 # ymm10 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm11, %ymm7 # ymm7 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7] + vpsrlq $0x20, %ymm4, %ymm4 + vpblendd $0xaa, %ymm11, %ymm4, %ymm11 # ymm11 = ymm4[0],ymm11[1],ymm4[2],ymm11[3],ymm4[4],ymm11[5],ymm4[6],ymm11[7] + vmovdqa 0x2e0(%rsi), %ymm15 + vmovdqa 0x300(%rsi), %ymm2 + vpmullw %ymm15, %ymm10, %ymm12 + vpmullw %ymm15, %ymm8, %ymm13 + vpmullw %ymm15, %ymm7, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm11, %ymm11 + vmovsldup %ymm9, %ymm4 # ymm4 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7] + vmovsldup %ymm5, %ymm6 # ymm6 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm3 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm8, %ymm9, %ymm4 + vpsubw %ymm8, %ymm9, %ymm8 + vpaddw %ymm7, %ymm6, %ymm9 + vpsubw %ymm7, %ymm6, %ymm7 + vpaddw %ymm11, %ymm5, %ymm6 + vpsubw %ymm11, %ymm5, %ymm11 + vpsubw %ymm12, %ymm3, %ymm3 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm8, %ymm8 + vpsubw %ymm14, %ymm9, %ymm9 + vpaddw %ymm14, %ymm7, %ymm7 + vpsubw %ymm15, %ymm6, %ymm6 + vpaddw %ymm15, %ymm11, %ymm11 + vpslld $0x10, %ymm7, %ymm5 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm7, %ymm9, %ymm7 # ymm7 = ymm9[0],ymm7[1],ymm9[2],ymm7[3],ymm9[4],ymm7[5],ymm9[6],ymm7[7],ymm9[8],ymm7[9],ymm9[10],ymm7[11],ymm9[12],ymm7[13],ymm9[14],ymm7[15] + vpslld $0x10, %ymm11, %ymm9 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa 0x320(%rsi), %ymm15 + vmovdqa 0x340(%rsi), %ymm2 + vpmullw %ymm15, %ymm5, %ymm12 + vpmullw %ymm15, %ymm7, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpslld $0x10, %ymm10, %ymm6 + vpblendw $0xaa, %ymm6, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm6[1],ymm3[2],ymm6[3],ymm3[4],ymm6[5],ymm3[6],ymm6[7],ymm3[8],ymm6[9],ymm3[10],ymm6[11],ymm3[12],ymm6[13],ymm3[14],ymm6[15] + vpsrld $0x10, %ymm3, %ymm3 + vpblendw $0xaa, %ymm10, %ymm3, %ymm10 # ymm10 = ymm3[0],ymm10[1],ymm3[2],ymm10[3],ymm3[4],ymm10[5],ymm3[6],ymm10[7],ymm3[8],ymm10[9],ymm3[10],ymm10[11],ymm3[12],ymm10[13],ymm3[14],ymm10[15] + vpslld $0x10, %ymm8, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm8[1],ymm4[2],ymm8[3],ymm4[4],ymm8[5],ymm4[6],ymm8[7],ymm4[8],ymm8[9],ymm4[10],ymm8[11],ymm4[12],ymm8[13],ymm4[14],ymm8[15] + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm5, %ymm6, %ymm4 + vpsubw %ymm5, %ymm6, %ymm5 + vpaddw %ymm7, %ymm10, %ymm6 + vpsubw %ymm7, %ymm10, %ymm7 + vpaddw %ymm9, %ymm3, %ymm10 + vpsubw %ymm9, %ymm3, %ymm9 + vpaddw %ymm11, %ymm8, %ymm3 + vpsubw %ymm11, %ymm8, %ymm11 + vpsubw %ymm12, %ymm4, %ymm4 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm13, %ymm6, %ymm6 + vpaddw %ymm13, %ymm7, %ymm7 + vpsubw %ymm14, %ymm10, %ymm10 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm3, %ymm3 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa 0x360(%rsi), %ymm14 + vmovdqa 0x3a0(%rsi), %ymm15 + vmovdqa 0x380(%rsi), %ymm8 + vmovdqa 0x3c0(%rsi), %ymm2 + vpmullw %ymm14, %ymm10, %ymm12 + vpmullw %ymm14, %ymm3, %ymm13 + vpmullw %ymm15, %ymm9, %ymm14 + vpmullw %ymm15, %ymm11, %ymm15 + vpmulhw %ymm8, %ymm10, %ymm10 + vpmulhw %ymm8, %ymm3, %ymm3 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm2, %ymm11, %ymm11 + vpmulhw %ymm0, %ymm12, %ymm12 + vpmulhw %ymm0, %ymm13, %ymm13 + vpmulhw %ymm0, %ymm14, %ymm14 + vpmulhw %ymm0, %ymm15, %ymm15 + vpaddw %ymm10, %ymm4, %ymm8 + vpsubw %ymm10, %ymm4, %ymm10 + vpaddw %ymm3, %ymm6, %ymm4 + vpsubw %ymm3, %ymm6, %ymm3 + vpaddw %ymm9, %ymm5, %ymm6 + vpsubw %ymm9, %ymm5, %ymm9 + vpaddw %ymm11, %ymm7, %ymm5 + vpsubw %ymm11, %ymm7, %ymm11 + vpsubw %ymm12, %ymm8, %ymm8 + vpaddw %ymm12, %ymm10, %ymm10 + vpsubw %ymm13, %ymm4, %ymm4 + vpaddw %ymm13, %ymm3, %ymm3 + vpsubw %ymm14, %ymm6, %ymm6 + vpaddw %ymm14, %ymm9, %ymm9 + vpsubw %ymm15, %ymm5, %ymm5 + vpaddw %ymm15, %ymm11, %ymm11 + vmovdqa %ymm8, 0x100(%rdi) + vmovdqa %ymm4, 0x120(%rdi) + vmovdqa %ymm10, 0x140(%rdi) + vmovdqa %ymm3, 0x160(%rdi) + vmovdqa %ymm6, 0x180(%rdi) + vmovdqa %ymm5, 0x1a0(%rdi) + vmovdqa %ymm9, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(ntt_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S new file mode 100644 index 0000000000..78b3b66d9a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttfrombytes_avx2_asm.S @@ -0,0 +1,217 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) +/*yaml + Name: nttfrombytes_avx2_asm + Description: x86_64 AVX2 polynomial deserialization in NTT domain + Signature: void mlk_nttfrombytes_avx2_asm(int16_t *r, const uint8_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 384 + permissions: read-only + c_parameter: const uint8_t *a + description: Input byte array (MLKEM_POLYBYTES = 384) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_nttfrombytes_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(nttfrombytes_avx2_asm) +MLK_ASM_FN_SYMBOL(nttfrombytes_avx2_asm) + + .cfi_startproc + movl $0xfff0fff, %eax # imm = 0xFFF0FFF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqu (%rsi), %ymm4 + vmovdqu 0x20(%rsi), %ymm5 + vmovdqu 0x40(%rsi), %ymm6 + vmovdqu 0x60(%rsi), %ymm7 + vmovdqu 0x80(%rsi), %ymm8 + vmovdqu 0xa0(%rsi), %ymm9 + vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1] + vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3] + vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3] + vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3] + vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3] + vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3] + vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7] + vpsrlq $0x20, %ymm8, %ymm8 + vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7] + vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7] + vpslld $0x10, %ymm7, %ymm10 + vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15] + vpslld $0x10, %ymm8, %ymm4 + vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15] + vpsrld $0x10, %ymm5, %ymm5 + vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15] + vpslld $0x10, %ymm9, %ymm5 + vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrlw $0xc, %ymm10, %ymm11 + vpsllw $0x4, %ymm7, %ymm12 + vpor %ymm11, %ymm12, %ymm11 + vpand %ymm0, %ymm10, %ymm10 + vpand %ymm0, %ymm11, %ymm11 + vpsrlw $0x8, %ymm7, %ymm12 + vpsllw $0x8, %ymm4, %ymm13 + vpor %ymm12, %ymm13, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpsrlw $0x4, %ymm4, %ymm13 + vpand %ymm0, %ymm13, %ymm13 + vpsrlw $0xc, %ymm8, %ymm14 + vpsllw $0x4, %ymm5, %ymm15 + vpor %ymm14, %ymm15, %ymm14 + vpand %ymm0, %ymm8, %ymm8 + vpand %ymm0, %ymm14, %ymm14 + vpsrlw $0x8, %ymm5, %ymm15 + vpsllw $0x8, %ymm9, %ymm1 + vpor %ymm15, %ymm1, %ymm15 + vpand %ymm0, %ymm15, %ymm15 + vpsrlw $0x4, %ymm9, %ymm1 + vpand %ymm0, %ymm1, %ymm1 + vmovdqa %ymm10, (%rdi) + vmovdqa %ymm11, 0x20(%rdi) + vmovdqa %ymm12, 0x40(%rdi) + vmovdqa %ymm13, 0x60(%rdi) + vmovdqa %ymm8, 0x80(%rdi) + vmovdqa %ymm14, 0xa0(%rdi) + vmovdqa %ymm15, 0xc0(%rdi) + vmovdqa %ymm1, 0xe0(%rdi) + vmovdqu 0xc0(%rsi), %ymm4 + vmovdqu 0xe0(%rsi), %ymm5 + vmovdqu 0x100(%rsi), %ymm6 + vmovdqu 0x120(%rsi), %ymm7 + vmovdqu 0x140(%rsi), %ymm8 + vmovdqu 0x160(%rsi), %ymm9 + vperm2i128 $0x20, %ymm7, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm7[0,1] + vperm2i128 $0x31, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[2,3],ymm7[2,3] + vperm2i128 $0x20, %ymm8, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[2,3],ymm9[2,3] + vpunpcklqdq %ymm8, %ymm3, %ymm6 # ymm6 = ymm3[0],ymm8[0],ymm3[2],ymm8[2] + vpunpckhqdq %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[1],ymm8[1],ymm3[3],ymm8[3] + vpunpcklqdq %ymm5, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm5[0],ymm7[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm7, %ymm5 # ymm5 = ymm7[1],ymm5[1],ymm7[3],ymm5[3] + vpunpcklqdq %ymm9, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm9[0],ymm4[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[1],ymm9[1],ymm4[3],ymm9[3] + vmovsldup %ymm5, %ymm4 # ymm4 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm7, %ymm6 # ymm6 = ymm7[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7] + vpsrlq $0x20, %ymm8, %ymm8 + vpblendd $0xaa, %ymm7, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7] + vmovsldup %ymm9, %ymm8 # ymm8 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm9, %ymm3, %ymm9 # ymm9 = ymm3[0],ymm9[1],ymm3[2],ymm9[3],ymm3[4],ymm9[5],ymm3[6],ymm9[7] + vpslld $0x10, %ymm7, %ymm10 + vpblendw $0xaa, %ymm10, %ymm4, %ymm10 # ymm10 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7],ymm4[8],ymm10[9],ymm4[10],ymm10[11],ymm4[12],ymm10[13],ymm4[14],ymm10[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm7, %ymm4, %ymm7 # ymm7 = ymm4[0],ymm7[1],ymm4[2],ymm7[3],ymm4[4],ymm7[5],ymm4[6],ymm7[7],ymm4[8],ymm7[9],ymm4[10],ymm7[11],ymm4[12],ymm7[13],ymm4[14],ymm7[15] + vpslld $0x10, %ymm8, %ymm4 + vpblendw $0xaa, %ymm4, %ymm5, %ymm4 # ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15] + vpsrld $0x10, %ymm5, %ymm5 + vpblendw $0xaa, %ymm8, %ymm5, %ymm8 # ymm8 = ymm5[0],ymm8[1],ymm5[2],ymm8[3],ymm5[4],ymm8[5],ymm5[6],ymm8[7],ymm5[8],ymm8[9],ymm5[10],ymm8[11],ymm5[12],ymm8[13],ymm5[14],ymm8[15] + vpslld $0x10, %ymm9, %ymm5 + vpblendw $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7],ymm6[8],ymm5[9],ymm6[10],ymm5[11],ymm6[12],ymm5[13],ymm6[14],ymm5[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm9, %ymm6, %ymm9 # ymm9 = ymm6[0],ymm9[1],ymm6[2],ymm9[3],ymm6[4],ymm9[5],ymm6[6],ymm9[7],ymm6[8],ymm9[9],ymm6[10],ymm9[11],ymm6[12],ymm9[13],ymm6[14],ymm9[15] + vpsrlw $0xc, %ymm10, %ymm11 + vpsllw $0x4, %ymm7, %ymm12 + vpor %ymm11, %ymm12, %ymm11 + vpand %ymm0, %ymm10, %ymm10 + vpand %ymm0, %ymm11, %ymm11 + vpsrlw $0x8, %ymm7, %ymm12 + vpsllw $0x8, %ymm4, %ymm13 + vpor %ymm12, %ymm13, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpsrlw $0x4, %ymm4, %ymm13 + vpand %ymm0, %ymm13, %ymm13 + vpsrlw $0xc, %ymm8, %ymm14 + vpsllw $0x4, %ymm5, %ymm15 + vpor %ymm14, %ymm15, %ymm14 + vpand %ymm0, %ymm8, %ymm8 + vpand %ymm0, %ymm14, %ymm14 + vpsrlw $0x8, %ymm5, %ymm15 + vpsllw $0x8, %ymm9, %ymm1 + vpor %ymm15, %ymm1, %ymm15 + vpand %ymm0, %ymm15, %ymm15 + vpsrlw $0x4, %ymm9, %ymm1 + vpand %ymm0, %ymm1, %ymm1 + vmovdqa %ymm10, 0x100(%rdi) + vmovdqa %ymm11, 0x120(%rdi) + vmovdqa %ymm12, 0x140(%rdi) + vmovdqa %ymm13, 0x160(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm14, 0x1a0(%rdi) + vmovdqa %ymm15, 0x1c0(%rdi) + vmovdqa %ymm1, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(nttfrombytes_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S new file mode 100644 index 0000000000..e2776b6ef2 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_ntttobytes_avx2_asm.S @@ -0,0 +1,205 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + !defined(MLK_CONFIG_NO_ENCAPS_API)) +/*yaml + Name: ntttobytes_avx2_asm + Description: x86_64 AVX2 polynomial serialization in NTT domain + Signature: void mlk_ntttobytes_avx2_asm(uint8_t *r, const int16_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 384 + permissions: write-only + c_parameter: uint8_t *r + description: Output byte array (MLKEM_POLYBYTES = 384) + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_ntttobytes_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(ntttobytes_avx2_asm) +MLK_ASM_FN_SYMBOL(ntttobytes_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa (%rsi), %ymm5 + vmovdqa 0x20(%rsi), %ymm6 + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vmovdqa 0x80(%rsi), %ymm9 + vmovdqa 0xa0(%rsi), %ymm10 + vmovdqa 0xc0(%rsi), %ymm11 + vmovdqa 0xe0(%rsi), %ymm12 + vpsllw $0xc, %ymm6, %ymm4 + vpor %ymm4, %ymm5, %ymm4 + vpsrlw $0x4, %ymm6, %ymm5 + vpsllw $0x8, %ymm7, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + vpsrlw $0x8, %ymm7, %ymm6 + vpsllw $0x4, %ymm8, %ymm7 + vpor %ymm6, %ymm7, %ymm6 + vpsllw $0xc, %ymm10, %ymm7 + vpor %ymm7, %ymm9, %ymm7 + vpsrlw $0x4, %ymm10, %ymm8 + vpsllw $0x8, %ymm11, %ymm9 + vpor %ymm8, %ymm9, %ymm8 + vpsrlw $0x8, %ymm11, %ymm9 + vpsllw $0x4, %ymm12, %ymm10 + vpor %ymm9, %ymm10, %ymm9 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3] + vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vmovdqu %ymm5, (%rdi) + vmovdqu %ymm7, 0x20(%rdi) + vmovdqu %ymm6, 0x40(%rdi) + vmovdqu %ymm8, 0x60(%rdi) + vmovdqu %ymm3, 0x80(%rdi) + vmovdqu %ymm9, 0xa0(%rdi) + vmovdqa 0x100(%rsi), %ymm5 + vmovdqa 0x120(%rsi), %ymm6 + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vmovdqa 0x180(%rsi), %ymm9 + vmovdqa 0x1a0(%rsi), %ymm10 + vmovdqa 0x1c0(%rsi), %ymm11 + vmovdqa 0x1e0(%rsi), %ymm12 + vpsllw $0xc, %ymm6, %ymm4 + vpor %ymm4, %ymm5, %ymm4 + vpsrlw $0x4, %ymm6, %ymm5 + vpsllw $0x8, %ymm7, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + vpsrlw $0x8, %ymm7, %ymm6 + vpsllw $0x4, %ymm8, %ymm7 + vpor %ymm6, %ymm7, %ymm6 + vpsllw $0xc, %ymm10, %ymm7 + vpor %ymm7, %ymm9, %ymm7 + vpsrlw $0x4, %ymm10, %ymm8 + vpsllw $0x8, %ymm11, %ymm9 + vpor %ymm8, %ymm9, %ymm8 + vpsrlw $0x8, %ymm11, %ymm9 + vpsllw $0x4, %ymm12, %ymm10 + vpor %ymm9, %ymm10, %ymm9 + vpslld $0x10, %ymm5, %ymm3 + vpblendw $0xaa, %ymm3, %ymm4, %ymm3 # ymm3 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7],ymm4[8],ymm3[9],ymm4[10],ymm3[11],ymm4[12],ymm3[13],ymm4[14],ymm3[15] + vpsrld $0x10, %ymm4, %ymm4 + vpblendw $0xaa, %ymm5, %ymm4, %ymm5 # ymm5 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7],ymm4[8],ymm5[9],ymm4[10],ymm5[11],ymm4[12],ymm5[13],ymm4[14],ymm5[15] + vpslld $0x10, %ymm7, %ymm4 + vpblendw $0xaa, %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3],ymm6[4],ymm4[5],ymm6[6],ymm4[7],ymm6[8],ymm4[9],ymm6[10],ymm4[11],ymm6[12],ymm4[13],ymm6[14],ymm4[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpslld $0x10, %ymm9, %ymm6 + vpblendw $0xaa, %ymm6, %ymm8, %ymm6 # ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7],ymm8[8],ymm6[9],ymm8[10],ymm6[11],ymm8[12],ymm6[13],ymm8[14],ymm6[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vmovsldup %ymm4, %ymm8 # ymm8 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm8, %ymm3, %ymm8 # ymm8 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4],ymm8[5],ymm3[6],ymm8[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm5, %ymm3 # ymm3 = ymm5[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7] + vpsrlq $0x20, %ymm6, %ymm6 + vpblendd $0xaa, %ymm5, %ymm6, %ymm5 # ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7] + vmovsldup %ymm9, %ymm6 # ymm6 = ymm9[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm6, %ymm7, %ymm6 # ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpunpcklqdq %ymm3, %ymm8, %ymm7 # ymm7 = ymm8[0],ymm3[0],ymm8[2],ymm3[2] + vpunpckhqdq %ymm3, %ymm8, %ymm3 # ymm3 = ymm8[1],ymm3[1],ymm8[3],ymm3[3] + vpunpcklqdq %ymm4, %ymm6, %ymm8 # ymm8 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] + vpunpckhqdq %ymm4, %ymm6, %ymm4 # ymm4 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] + vpunpcklqdq %ymm9, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm9[0],ymm5[2],ymm9[2] + vpunpckhqdq %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[1],ymm9[1],ymm5[3],ymm9[3] + vperm2i128 $0x20, %ymm8, %ymm7, %ymm5 # ymm5 = ymm7[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm3, %ymm6, %ymm7 # ymm7 = ymm6[0,1],ymm3[0,1] + vperm2i128 $0x31, %ymm3, %ymm6, %ymm3 # ymm3 = ymm6[2,3],ymm3[2,3] + vperm2i128 $0x20, %ymm9, %ymm4, %ymm6 # ymm6 = ymm4[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm4, %ymm9 # ymm9 = ymm4[2,3],ymm9[2,3] + vmovdqu %ymm5, 0xc0(%rdi) + vmovdqu %ymm7, 0xe0(%rdi) + vmovdqu %ymm6, 0x100(%rdi) + vmovdqu %ymm8, 0x120(%rdi) + vmovdqu %ymm3, 0x140(%rdi) + vmovdqu %ymm9, 0x160(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(ntttobytes_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S new file mode 100644 index 0000000000..015d43d9c7 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_nttunpack_avx2_asm.S @@ -0,0 +1,190 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: nttunpack_avx2_asm + Description: x86_64 AVX2 NTT unpack from custom to bitreversed order + Signature: void mlk_nttunpack_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_nttunpack_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(nttunpack_avx2_asm) +MLK_ASM_FN_SYMBOL(nttunpack_avx2_asm) + + .cfi_startproc + vmovdqa (%rdi), %ymm4 + vmovdqa 0x20(%rdi), %ymm5 + vmovdqa 0x40(%rdi), %ymm6 + vmovdqa 0x60(%rdi), %ymm7 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm9 + vmovdqa 0xc0(%rdi), %ymm10 + vmovdqa 0xe0(%rdi), %ymm11 + vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3] + vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3] + vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3] + vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3] + vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2] + vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3] + vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3] + vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7] + vpsrlq $0x20, %ymm10, %ymm10 + vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7] + vpslld $0x10, %ymm5, %ymm10 + vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpslld $0x10, %ymm4, %ymm9 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] + vpslld $0x10, %ymm3, %ymm8 + vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15] + vpsrld $0x10, %ymm7, %ymm7 + vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15] + vpslld $0x10, %ymm11, %ymm7 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa %ymm10, (%rdi) + vmovdqa %ymm5, 0x20(%rdi) + vmovdqa %ymm9, 0x40(%rdi) + vmovdqa %ymm4, 0x60(%rdi) + vmovdqa %ymm8, 0x80(%rdi) + vmovdqa %ymm3, 0xa0(%rdi) + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm11, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm4 + vmovdqa 0x120(%rdi), %ymm5 + vmovdqa 0x140(%rdi), %ymm6 + vmovdqa 0x160(%rdi), %ymm7 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm9 + vmovdqa 0x1c0(%rdi), %ymm10 + vmovdqa 0x1e0(%rdi), %ymm11 + vperm2i128 $0x20, %ymm8, %ymm4, %ymm3 # ymm3 = ymm4[0,1],ymm8[0,1] + vperm2i128 $0x31, %ymm8, %ymm4, %ymm8 # ymm8 = ymm4[2,3],ymm8[2,3] + vperm2i128 $0x20, %ymm9, %ymm5, %ymm4 # ymm4 = ymm5[0,1],ymm9[0,1] + vperm2i128 $0x31, %ymm9, %ymm5, %ymm9 # ymm9 = ymm5[2,3],ymm9[2,3] + vperm2i128 $0x20, %ymm10, %ymm6, %ymm5 # ymm5 = ymm6[0,1],ymm10[0,1] + vperm2i128 $0x31, %ymm10, %ymm6, %ymm10 # ymm10 = ymm6[2,3],ymm10[2,3] + vperm2i128 $0x20, %ymm11, %ymm7, %ymm6 # ymm6 = ymm7[0,1],ymm11[0,1] + vperm2i128 $0x31, %ymm11, %ymm7, %ymm11 # ymm11 = ymm7[2,3],ymm11[2,3] + vpunpcklqdq %ymm5, %ymm3, %ymm7 # ymm7 = ymm3[0],ymm5[0],ymm3[2],ymm5[2] + vpunpckhqdq %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[1],ymm5[1],ymm3[3],ymm5[3] + vpunpcklqdq %ymm10, %ymm8, %ymm3 # ymm3 = ymm8[0],ymm10[0],ymm8[2],ymm10[2] + vpunpckhqdq %ymm10, %ymm8, %ymm10 # ymm10 = ymm8[1],ymm10[1],ymm8[3],ymm10[3] + vpunpcklqdq %ymm6, %ymm4, %ymm8 # ymm8 = ymm4[0],ymm6[0],ymm4[2],ymm6[2] + vpunpckhqdq %ymm6, %ymm4, %ymm6 # ymm6 = ymm4[1],ymm6[1],ymm4[3],ymm6[3] + vpunpcklqdq %ymm11, %ymm9, %ymm4 # ymm4 = ymm9[0],ymm11[0],ymm9[2],ymm11[2] + vpunpckhqdq %ymm11, %ymm9, %ymm11 # ymm11 = ymm9[1],ymm11[1],ymm9[3],ymm11[3] + vmovsldup %ymm8, %ymm9 # ymm9 = ymm8[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm9, %ymm7, %ymm9 # ymm9 = ymm7[0],ymm9[1],ymm7[2],ymm9[3],ymm7[4],ymm9[5],ymm7[6],ymm9[7] + vpsrlq $0x20, %ymm7, %ymm7 + vpblendd $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7] + vmovsldup %ymm6, %ymm7 # ymm7 = ymm6[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm7, %ymm5, %ymm7 # ymm7 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7] + vpsrlq $0x20, %ymm5, %ymm5 + vpblendd $0xaa, %ymm6, %ymm5, %ymm6 # ymm6 = ymm5[0],ymm6[1],ymm5[2],ymm6[3],ymm5[4],ymm6[5],ymm5[6],ymm6[7] + vmovsldup %ymm4, %ymm5 # ymm5 = ymm4[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm5, %ymm3, %ymm5 # ymm5 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7] + vpsrlq $0x20, %ymm3, %ymm3 + vpblendd $0xaa, %ymm4, %ymm3, %ymm4 # ymm4 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7] + vmovsldup %ymm11, %ymm3 # ymm3 = ymm11[0,0,2,2,4,4,6,6] + vpblendd $0xaa, %ymm3, %ymm10, %ymm3 # ymm3 = ymm10[0],ymm3[1],ymm10[2],ymm3[3],ymm10[4],ymm3[5],ymm10[6],ymm3[7] + vpsrlq $0x20, %ymm10, %ymm10 + vpblendd $0xaa, %ymm11, %ymm10, %ymm11 # ymm11 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7] + vpslld $0x10, %ymm5, %ymm10 + vpblendw $0xaa, %ymm10, %ymm9, %ymm10 # ymm10 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4],ymm10[5],ymm9[6],ymm10[7],ymm9[8],ymm10[9],ymm9[10],ymm10[11],ymm9[12],ymm10[13],ymm9[14],ymm10[15] + vpsrld $0x10, %ymm9, %ymm9 + vpblendw $0xaa, %ymm5, %ymm9, %ymm5 # ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7],ymm9[8],ymm5[9],ymm9[10],ymm5[11],ymm9[12],ymm5[13],ymm9[14],ymm5[15] + vpslld $0x10, %ymm4, %ymm9 + vpblendw $0xaa, %ymm9, %ymm8, %ymm9 # ymm9 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7],ymm8[8],ymm9[9],ymm8[10],ymm9[11],ymm8[12],ymm9[13],ymm8[14],ymm9[15] + vpsrld $0x10, %ymm8, %ymm8 + vpblendw $0xaa, %ymm4, %ymm8, %ymm4 # ymm4 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4],ymm4[5],ymm8[6],ymm4[7],ymm8[8],ymm4[9],ymm8[10],ymm4[11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] + vpslld $0x10, %ymm3, %ymm8 + vpblendw $0xaa, %ymm8, %ymm7, %ymm8 # ymm8 = ymm7[0],ymm8[1],ymm7[2],ymm8[3],ymm7[4],ymm8[5],ymm7[6],ymm8[7],ymm7[8],ymm8[9],ymm7[10],ymm8[11],ymm7[12],ymm8[13],ymm7[14],ymm8[15] + vpsrld $0x10, %ymm7, %ymm7 + vpblendw $0xaa, %ymm3, %ymm7, %ymm3 # ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4],ymm3[5],ymm7[6],ymm3[7],ymm7[8],ymm3[9],ymm7[10],ymm3[11],ymm7[12],ymm3[13],ymm7[14],ymm3[15] + vpslld $0x10, %ymm11, %ymm7 + vpblendw $0xaa, %ymm7, %ymm6, %ymm7 # ymm7 = ymm6[0],ymm7[1],ymm6[2],ymm7[3],ymm6[4],ymm7[5],ymm6[6],ymm7[7],ymm6[8],ymm7[9],ymm6[10],ymm7[11],ymm6[12],ymm7[13],ymm6[14],ymm7[15] + vpsrld $0x10, %ymm6, %ymm6 + vpblendw $0xaa, %ymm11, %ymm6, %ymm11 # ymm11 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4],ymm11[5],ymm6[6],ymm11[7],ymm6[8],ymm11[9],ymm6[10],ymm11[11],ymm6[12],ymm11[13],ymm6[14],ymm11[15] + vmovdqa %ymm10, 0x100(%rdi) + vmovdqa %ymm5, 0x120(%rdi) + vmovdqa %ymm9, 0x140(%rdi) + vmovdqa %ymm4, 0x160(%rdi) + vmovdqa %ymm8, 0x180(%rdi) + vmovdqa %ymm3, 0x1a0(%rdi) + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm11, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(nttunpack_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S new file mode 100644 index 0000000000..af8cecfd9c --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S @@ -0,0 +1,413 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d10_avx2_asm + * + * Description: Compression of a polynomial to 10 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D10) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_compress_d10_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=10) + Signature: void mlk_poly_compress_d10_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 320 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d10_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d10_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d10_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpsllw $0x3, %ymm0, %ymm1 + movl $0xf000f, %eax # imm = 0xF000F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x10001000, %eax # imm = 0x10001000 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x3ff03ff, %eax # imm = 0x3FF03FF + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movabsq $0x400000104000001, %rax # imm = 0x400000104000001 + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + movl $0xc, %eax + vmovq %rax, %xmm6 + vpbroadcastq %xmm6, %ymm6 + vmovdqa (%rdx), %ymm7 + vmovdqa (%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, (%rdi) + vmovd %xmm9, 0x10(%rdi) + vmovdqa 0x20(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x14(%rdi) + vmovd %xmm9, 0x24(%rdi) + vmovdqa 0x40(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x28(%rdi) + vmovd %xmm9, 0x38(%rdi) + vmovdqa 0x60(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x3c(%rdi) + vmovd %xmm9, 0x4c(%rdi) + vmovdqa 0x80(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x50(%rdi) + vmovd %xmm9, 0x60(%rdi) + vmovdqa 0xa0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x64(%rdi) + vmovd %xmm9, 0x74(%rdi) + vmovdqa 0xc0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x78(%rdi) + vmovd %xmm9, 0x88(%rdi) + vmovdqa 0xe0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x8c(%rdi) + vmovd %xmm9, 0x9c(%rdi) + vmovdqa 0x100(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xa0(%rdi) + vmovd %xmm9, 0xb0(%rdi) + vmovdqa 0x120(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xb4(%rdi) + vmovd %xmm9, 0xc4(%rdi) + vmovdqa 0x140(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xc8(%rdi) + vmovd %xmm9, 0xd8(%rdi) + vmovdqa 0x160(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xdc(%rdi) + vmovd %xmm9, 0xec(%rdi) + vmovdqa 0x180(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0xf0(%rdi) + vmovd %xmm9, 0x100(%rdi) + vmovdqa 0x1a0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x104(%rdi) + vmovd %xmm9, 0x114(%rdi) + vmovdqa 0x1c0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x118(%rdi) + vmovd %xmm9, 0x128(%rdi) + vmovdqa 0x1e0(%rsi), %ymm8 + vpmullw %ymm1, %ymm8, %ymm9 + vpaddw %ymm2, %ymm8, %ymm10 + vpsllw $0x3, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm8, %ymm8 + vpsubw %ymm10, %ymm9, %ymm10 + vpandn %ymm10, %ymm9, %ymm9 + vpsrlw $0xf, %ymm9, %ymm9 + vpsubw %ymm9, %ymm8, %ymm8 + vpmulhrsw %ymm3, %ymm8, %ymm8 + vpand %ymm4, %ymm8, %ymm8 + vpmaddwd %ymm5, %ymm8, %ymm8 + vpsllvd %ymm6, %ymm8, %ymm8 + vpsrlq $0xc, %ymm8, %ymm8 + vpshufb %ymm7, %ymm8, %ymm8 + vextracti128 $0x1, %ymm8, %xmm9 + vpblendw $0xe0, %xmm9, %xmm8, %xmm8 # xmm8 = xmm8[0,1,2,3,4],xmm9[5,6,7] + vmovdqu %xmm8, 0x12c(%rdi) + vmovd %xmm9, 0x13c(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d10_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S new file mode 100644 index 0000000000..9442315c3b --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S @@ -0,0 +1,479 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d11_avx2_asm + * + * Description: Compression of a polynomial to 11 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D11) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to constants + * (srlvqidx[0:32], shufbidx[32:64]) + **************************************************/ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_compress_d11_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=11) + Signature: void mlk_poly_compress_d11_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 352 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 64 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d11_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d11_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d11_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vpsllw $0x3, %ymm0, %ymm1 + movl $0x240024, %eax # imm = 0x240024 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x20002000, %eax # imm = 0x20002000 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x7ff07ff, %eax # imm = 0x7FF07FF + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movabsq $0x800000108000001, %rax # imm = 0x800000108000001 + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + movl $0xa, %eax + vmovq %rax, %xmm6 + vpbroadcastq %xmm6, %ymm6 + vmovdqa (%rdx), %ymm7 + vmovdqa 0x20(%rdx), %ymm8 + vmovdqa (%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, (%rdi) + vmovd %xmm10, 0x10(%rdi) + vpextrw $0x2, %xmm10, 0x14(%rdi) + vmovdqa 0x20(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x16(%rdi) + vmovd %xmm10, 0x26(%rdi) + vpextrw $0x2, %xmm10, 0x2a(%rdi) + vmovdqa 0x40(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x2c(%rdi) + vmovd %xmm10, 0x3c(%rdi) + vpextrw $0x2, %xmm10, 0x40(%rdi) + vmovdqa 0x60(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x42(%rdi) + vmovd %xmm10, 0x52(%rdi) + vpextrw $0x2, %xmm10, 0x56(%rdi) + vmovdqa 0x80(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x58(%rdi) + vmovd %xmm10, 0x68(%rdi) + vpextrw $0x2, %xmm10, 0x6c(%rdi) + vmovdqa 0xa0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x6e(%rdi) + vmovd %xmm10, 0x7e(%rdi) + vpextrw $0x2, %xmm10, 0x82(%rdi) + vmovdqa 0xc0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x84(%rdi) + vmovd %xmm10, 0x94(%rdi) + vpextrw $0x2, %xmm10, 0x98(%rdi) + vmovdqa 0xe0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x9a(%rdi) + vmovd %xmm10, 0xaa(%rdi) + vpextrw $0x2, %xmm10, 0xae(%rdi) + vmovdqa 0x100(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xb0(%rdi) + vmovd %xmm10, 0xc0(%rdi) + vpextrw $0x2, %xmm10, 0xc4(%rdi) + vmovdqa 0x120(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xc6(%rdi) + vmovd %xmm10, 0xd6(%rdi) + vpextrw $0x2, %xmm10, 0xda(%rdi) + vmovdqa 0x140(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xdc(%rdi) + vmovd %xmm10, 0xec(%rdi) + vpextrw $0x2, %xmm10, 0xf0(%rdi) + vmovdqa 0x160(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0xf2(%rdi) + vmovd %xmm10, 0x102(%rdi) + vpextrw $0x2, %xmm10, 0x106(%rdi) + vmovdqa 0x180(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x108(%rdi) + vmovd %xmm10, 0x118(%rdi) + vpextrw $0x2, %xmm10, 0x11c(%rdi) + vmovdqa 0x1a0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x11e(%rdi) + vmovd %xmm10, 0x12e(%rdi) + vpextrw $0x2, %xmm10, 0x132(%rdi) + vmovdqa 0x1c0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x134(%rdi) + vmovd %xmm10, 0x144(%rdi) + vpextrw $0x2, %xmm10, 0x148(%rdi) + vmovdqa 0x1e0(%rsi), %ymm9 + vpmullw %ymm1, %ymm9, %ymm10 + vpaddw %ymm2, %ymm9, %ymm11 + vpsllw $0x3, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm9, %ymm9 + vpsubw %ymm11, %ymm10, %ymm11 + vpandn %ymm11, %ymm10, %ymm10 + vpsrlw $0xf, %ymm10, %ymm10 + vpsubw %ymm10, %ymm9, %ymm9 + vpmulhrsw %ymm3, %ymm9, %ymm9 + vpand %ymm4, %ymm9, %ymm9 + vpmaddwd %ymm5, %ymm9, %ymm9 + vpsllvd %ymm6, %ymm9, %ymm9 + vpsrldq $0x8, %ymm9, %ymm10 # ymm10 = ymm9[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero + vpsrlvq %ymm7, %ymm9, %ymm9 + vpsllq $0x22, %ymm10, %ymm10 + vpaddq %ymm10, %ymm9, %ymm9 + vpshufb %ymm8, %ymm9, %ymm9 + vextracti128 $0x1, %ymm9, %xmm10 + vpblendvb %xmm8, %xmm10, %xmm9, %xmm9 + vmovdqu %xmm9, 0x14a(%rdi) + vmovd %xmm10, 0x15a(%rdi) + vpextrw $0x2, %xmm10, 0x15e(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d11_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S new file mode 100644 index 0000000000..428fda5b42 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S @@ -0,0 +1,194 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d4_avx2_asm + * + * Description: Compression of a polynomial to 4 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D4) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to permdidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_compress_d4_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=4) + Signature: void mlk_poly_compress_d4_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 128 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d4_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d4_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x2000200, %eax # imm = 0x2000200 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0xf000f, %eax # imm = 0xF000F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x10011001, %eax # imm = 0x10011001 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa (%rsi), %ymm5 + vmovdqa 0x20(%rsi), %ymm6 + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, (%rdi) + vmovdqa 0x80(%rsi), %ymm5 + vmovdqa 0xa0(%rsi), %ymm6 + vmovdqa 0xc0(%rsi), %ymm7 + vmovdqa 0xe0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x20(%rdi) + vmovdqa 0x100(%rsi), %ymm5 + vmovdqa 0x120(%rsi), %ymm6 + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x40(%rdi) + vmovdqa 0x180(%rsi), %ymm5 + vmovdqa 0x1a0(%rsi), %ymm6 + vmovdqa 0x1c0(%rsi), %ymm7 + vmovdqa 0x1e0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm5, %ymm5 + vpmulhrsw %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm5, %ymm5 + vpand %ymm2, %ymm6, %ymm6 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm6, %ymm5, %ymm5 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm5, %ymm5 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpackuswb %ymm7, %ymm5, %ymm5 + vpermd %ymm5, %ymm4, %ymm5 + vmovdqu %ymm5, 0x60(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2 || MLKEM_K == 3) \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S new file mode 100644 index 0000000000..41e3c679d7 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S @@ -0,0 +1,251 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_compress_d5_avx2_asm + * + * Description: Compression of a polynomial to 5 bits per coefficient. + * + * Arguments: - uint8_t *r: pointer to output byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D5) + * - const int16_t *a: pointer to input polynomial + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API)) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_compress_d5_avx2_asm + Description: x86_64 AVX2 polynomial compression (d=5) + Signature: void mlk_poly_compress_d5_avx2_asm(uint8_t *r, const int16_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 160 + permissions: write-only + c_parameter: uint8_t *r + description: Output compressed polynomial + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *a + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed compression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_compress_d5_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_compress_d5_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_compress_d5_avx2_asm) + + .cfi_startproc + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4000400, %eax # imm = 0x4000400 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x1f001f, %eax # imm = 0x1F001F + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x20012001, %eax # imm = 0x20012001 + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x4000001, %eax # imm = 0x4000001 + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movl $0xc, %eax + vmovq %rax, %xmm5 + vpbroadcastq %xmm5, %ymm5 + vmovdqa (%rdx), %ymm6 + vmovdqa (%rsi), %ymm7 + vmovdqa 0x20(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, (%rdi) + vmovd %xmm8, 0x10(%rdi) + vmovdqa 0x40(%rsi), %ymm7 + vmovdqa 0x60(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x14(%rdi) + vmovd %xmm8, 0x24(%rdi) + vmovdqa 0x80(%rsi), %ymm7 + vmovdqa 0xa0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x28(%rdi) + vmovd %xmm8, 0x38(%rdi) + vmovdqa 0xc0(%rsi), %ymm7 + vmovdqa 0xe0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x3c(%rdi) + vmovd %xmm8, 0x4c(%rdi) + vmovdqa 0x100(%rsi), %ymm7 + vmovdqa 0x120(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x50(%rdi) + vmovd %xmm8, 0x60(%rdi) + vmovdqa 0x140(%rsi), %ymm7 + vmovdqa 0x160(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x64(%rdi) + vmovd %xmm8, 0x74(%rdi) + vmovdqa 0x180(%rsi), %ymm7 + vmovdqa 0x1a0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x78(%rdi) + vmovd %xmm8, 0x88(%rdi) + vmovdqa 0x1c0(%rsi), %ymm7 + vmovdqa 0x1e0(%rsi), %ymm8 + vpmulhw %ymm0, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm8, %ymm8 + vpmulhrsw %ymm1, %ymm7, %ymm7 + vpmulhrsw %ymm1, %ymm8, %ymm8 + vpand %ymm2, %ymm7, %ymm7 + vpand %ymm2, %ymm8, %ymm8 + vpackuswb %ymm8, %ymm7, %ymm7 + vpmaddubsw %ymm3, %ymm7, %ymm7 + vpmaddwd %ymm4, %ymm7, %ymm7 + vpsllvd %ymm5, %ymm7, %ymm7 + vpsrlvq %ymm5, %ymm7, %ymm7 + vpshufb %ymm6, %ymm7, %ymm7 + vextracti128 $0x1, %ymm7, %xmm8 + vpblendvb %xmm6, %xmm8, %xmm7, %xmm7 + vmovdqu %xmm7, 0x8c(%rdi) + vmovd %xmm8, 0x9c(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_compress_d5_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API) && \ + (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S new file mode 100644 index 0000000000..656955555a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S @@ -0,0 +1,257 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d10_avx2_asm + * + * Description: Decompression of a polynomial from 10 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D10) + * - const uint8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_decompress_d10_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=10) + Signature: void mlk_poly_decompress_d10_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 320 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d10_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d10_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d10_avx2_asm) + + .cfi_startproc + movl $0xd013404, %eax # imm = 0xD013404 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4, %eax + vmovq %rax, %xmm1 + vpbroadcastq %xmm1, %ymm1 + movl $0x7fe01ff8, %eax # imm = 0x7FE01FF8 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdx), %ymm3 + vmovdqu (%rsi), %xmm4 + vmovd 0x10(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovdqu 0x14(%rsi), %xmm4 + vmovd 0x24(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovdqu 0x28(%rsi), %xmm4 + vmovd 0x38(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovdqu 0x3c(%rsi), %xmm4 + vmovd 0x4c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovdqu 0x50(%rsi), %xmm4 + vmovd 0x60(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovdqu 0x64(%rsi), %xmm4 + vmovd 0x74(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovdqu 0x78(%rsi), %xmm4 + vmovd 0x88(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovdqu 0x8c(%rsi), %xmm4 + vmovd 0x9c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovdqu 0xa0(%rsi), %xmm4 + vmovd 0xb0(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovdqu 0xb4(%rsi), %xmm4 + vmovd 0xc4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovdqu 0xc8(%rsi), %xmm4 + vmovd 0xd8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovdqu 0xdc(%rsi), %xmm4 + vmovd 0xec(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovdqu 0xf0(%rsi), %xmm4 + vmovd 0x100(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovdqu 0x104(%rsi), %xmm4 + vmovd 0x114(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovdqu 0x118(%rsi), %xmm4 + vmovd 0x128(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovdqu 0x12c(%rsi), %xmm4 + vmovd 0x13c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm4, %ymm4 + vpermq $0x94, %ymm4, %ymm4 # ymm4 = ymm4[0,1,1,2] + vpshufb %ymm3, %ymm4, %ymm4 + vpsllvd %ymm1, %ymm4, %ymm4 + vpsrlw $0x1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d10_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 2 || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S new file mode 100644 index 0000000000..d5c42312fc --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S @@ -0,0 +1,307 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d11_avx2_asm + * + * Description: Decompression of a polynomial from 11 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D11) + * - const uint8_t *data: pointer to constants + * (shufbidx[0:32], srlvdidx[32:64], + * srlvqidx[64:96], shift[96:128]) + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_decompress_d11_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=11) + Signature: void mlk_poly_decompress_d11_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 352 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 128 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d11_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d11_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d11_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x7ff07ff0, %eax # imm = 0x7FF07FF0 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rdx), %ymm2 + vmovdqa 0x20(%rdx), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqu (%rsi), %xmm6 + vmovd 0x10(%rsi), %xmm7 + vpinsrw $0x2, 0x14(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, (%rdi) + vmovdqu 0x16(%rsi), %xmm6 + vmovd 0x26(%rsi), %xmm7 + vpinsrw $0x2, 0x2a(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x20(%rdi) + vmovdqu 0x2c(%rsi), %xmm6 + vmovd 0x3c(%rsi), %xmm7 + vpinsrw $0x2, 0x40(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x40(%rdi) + vmovdqu 0x42(%rsi), %xmm6 + vmovd 0x52(%rsi), %xmm7 + vpinsrw $0x2, 0x56(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x60(%rdi) + vmovdqu 0x58(%rsi), %xmm6 + vmovd 0x68(%rsi), %xmm7 + vpinsrw $0x2, 0x6c(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x80(%rdi) + vmovdqu 0x6e(%rsi), %xmm6 + vmovd 0x7e(%rsi), %xmm7 + vpinsrw $0x2, 0x82(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xa0(%rdi) + vmovdqu 0x84(%rsi), %xmm6 + vmovd 0x94(%rsi), %xmm7 + vpinsrw $0x2, 0x98(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xc0(%rdi) + vmovdqu 0x9a(%rsi), %xmm6 + vmovd 0xaa(%rsi), %xmm7 + vpinsrw $0x2, 0xae(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0xe0(%rdi) + vmovdqu 0xb0(%rsi), %xmm6 + vmovd 0xc0(%rsi), %xmm7 + vpinsrw $0x2, 0xc4(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x100(%rdi) + vmovdqu 0xc6(%rsi), %xmm6 + vmovd 0xd6(%rsi), %xmm7 + vpinsrw $0x2, 0xda(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x120(%rdi) + vmovdqu 0xdc(%rsi), %xmm6 + vmovd 0xec(%rsi), %xmm7 + vpinsrw $0x2, 0xf0(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x140(%rdi) + vmovdqu 0xf2(%rsi), %xmm6 + vmovd 0x102(%rsi), %xmm7 + vpinsrw $0x2, 0x106(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x160(%rdi) + vmovdqu 0x108(%rsi), %xmm6 + vmovd 0x118(%rsi), %xmm7 + vpinsrw $0x2, 0x11c(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x180(%rdi) + vmovdqu 0x11e(%rsi), %xmm6 + vmovd 0x12e(%rsi), %xmm7 + vpinsrw $0x2, 0x132(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1a0(%rdi) + vmovdqu 0x134(%rsi), %xmm6 + vmovd 0x144(%rsi), %xmm7 + vpinsrw $0x2, 0x148(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1c0(%rdi) + vmovdqu 0x14a(%rsi), %xmm6 + vmovd 0x15a(%rsi), %xmm7 + vpinsrw $0x2, 0x15e(%rsi), %xmm7, %xmm7 + vinserti128 $0x1, %xmm7, %ymm6, %ymm6 + vpermq $0x94, %ymm6, %ymm6 # ymm6 = ymm6[0,1,1,2] + vpshufb %ymm2, %ymm6, %ymm6 + vpsrlvd %ymm3, %ymm6, %ymm6 + vpsrlvq %ymm4, %ymm6, %ymm6 + vpmullw %ymm5, %ymm6, %ymm6 + vpsrlw $0x1, %ymm6, %ymm6 + vpand %ymm1, %ymm6, %ymm6 + vpmulhrsw %ymm0, %ymm6, %ymm6 + vmovdqu %ymm6, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d11_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S new file mode 100644 index 0000000000..fd9bbc87e1 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S @@ -0,0 +1,209 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d4_avx2_asm + * + * Description: Decompression of a polynomial from 4 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D4) + * - const int8_t *data: pointer to shufbidx constant + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2 || MLKEM_K == 3) +/*yaml + Name: poly_decompress_d4_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=4) + Signature: void mlk_poly_decompress_d4_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 128 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 32 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d4_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d4_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf0000f, %eax # imm = 0xF0000F + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x800800, %eax # imm = 0x800800 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdx), %ymm3 + vmovq (%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovq 0x8(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovq 0x10(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovq 0x18(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovq 0x20(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovq 0x28(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovq 0x30(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovq 0x38(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovq 0x40(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovq 0x48(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovq 0x50(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovq 0x58(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovq 0x60(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovq 0x68(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovq 0x70(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovq 0x78(%rsi), %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm3, %ymm4, %ymm4 + vpand %ymm1, %ymm4, %ymm4 + vpmullw %ymm2, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 2 || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S new file mode 100644 index 0000000000..46aa87c8ec --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S @@ -0,0 +1,222 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + */ + +/************************************************* + * Name: mlk_poly_decompress_d5_avx2_asm + * + * Description: Decompression of a polynomial from 5 bits per coefficient. + * + * Arguments: - int16_t *r: pointer to output polynomial + * - const uint8_t *a: pointer to input byte array + * (of length MLKEM_POLYCOMPRESSEDBYTES_D5) + * - const uint8_t *data: pointer to constants + * (shufbidx[0:32], mask[32:64], shift[64:96]) + **************************************************/ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_DECAPS_API) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: poly_decompress_d5_avx2_asm + Description: x86_64 AVX2 polynomial decompression (d=5) + Signature: void mlk_poly_decompress_d5_avx2_asm(int16_t *r, const uint8_t *a, const uint8_t *data) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 160 + permissions: read-only + c_parameter: const uint8_t *a + description: Input compressed polynomial + rdx: + type: buffer + size_bytes: 96 + permissions: read-only + c_parameter: const uint8_t *data + description: Precomputed decompression constants +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_decompress_d5_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_decompress_d5_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_decompress_d5_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa (%rdx), %ymm1 + vmovdqa 0x20(%rdx), %ymm2 + vmovdqa 0x40(%rdx), %ymm3 + vmovq (%rsi), %xmm4 + vpinsrw $0x4, 0x8(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, (%rdi) + vmovq 0xa(%rsi), %xmm4 + vpinsrw $0x4, 0x12(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x20(%rdi) + vmovq 0x14(%rsi), %xmm4 + vpinsrw $0x4, 0x1c(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x40(%rdi) + vmovq 0x1e(%rsi), %xmm4 + vpinsrw $0x4, 0x26(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x60(%rdi) + vmovq 0x28(%rsi), %xmm4 + vpinsrw $0x4, 0x30(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x80(%rdi) + vmovq 0x32(%rsi), %xmm4 + vpinsrw $0x4, 0x3a(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xa0(%rdi) + vmovq 0x3c(%rsi), %xmm4 + vpinsrw $0x4, 0x44(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xc0(%rdi) + vmovq 0x46(%rsi), %xmm4 + vpinsrw $0x4, 0x4e(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0xe0(%rdi) + vmovq 0x50(%rsi), %xmm4 + vpinsrw $0x4, 0x58(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x100(%rdi) + vmovq 0x5a(%rsi), %xmm4 + vpinsrw $0x4, 0x62(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x120(%rdi) + vmovq 0x64(%rsi), %xmm4 + vpinsrw $0x4, 0x6c(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x140(%rdi) + vmovq 0x6e(%rsi), %xmm4 + vpinsrw $0x4, 0x76(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x160(%rdi) + vmovq 0x78(%rsi), %xmm4 + vpinsrw $0x4, 0x80(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x180(%rdi) + vmovq 0x82(%rsi), %xmm4 + vpinsrw $0x4, 0x8a(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1a0(%rdi) + vmovq 0x8c(%rsi), %xmm4 + vpinsrw $0x4, 0x94(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1c0(%rdi) + vmovq 0x96(%rsi), %xmm4 + vpinsrw $0x4, 0x9e(%rsi), %xmm4, %xmm4 + vinserti128 $0x1, %xmm4, %ymm4, %ymm4 + vpshufb %ymm1, %ymm4, %ymm4 + vpand %ymm2, %ymm4, %ymm4 + vpmullw %ymm3, %ymm4, %ymm4 + vpmulhrsw %ymm0, %ymm4, %ymm4 + vmovdqu %ymm4, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_decompress_d5_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_DECAPS_API && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || \ + MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S new file mode 100644 index 0000000000..23a095a732 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S @@ -0,0 +1,118 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: poly_mulcache_compute_avx2_asm + Description: x86_64 AVX2 mulcache computation + Signature: void mlk_poly_mulcache_compute_avx2_asm(int16_t *out, const int16_t *in, const int16_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 256 + permissions: write-only + c_parameter: int16_t *out + description: Output mulcache (128 x int16_t) + rsi: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *in + description: Input polynomial (256 x int16_t) + rdx: + type: buffer + size_bytes: 1248 + permissions: read-only + c_parameter: const int16_t *qdata + description: Precomputed constants (624 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_poly_mulcache_compute_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(poly_mulcache_compute_avx2_asm) +MLK_ASM_FN_SYMBOL(poly_mulcache_compute_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + vmovdqa 0x20(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x3e0(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm8, 0x20(%rdi) + vmovdqa 0xa0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x480(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm8, 0x60(%rdi) + vmovdqa 0x120(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x420(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm8, 0xa0(%rdi) + vmovdqa 0x1a0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x4c0(%rdx), %ymm1 + vpmullw %ymm2, %ymm1, %ymm5 + vpmullw %ymm3, %ymm1, %ymm6 + vpmulhw %ymm2, %ymm4, %ymm7 + vpmulhw %ymm3, %ymm4, %ymm8 + vpmulhw %ymm5, %ymm0, %ymm9 + vpmulhw %ymm6, %ymm0, %ymm10 + vpsubw %ymm9, %ymm7, %ymm7 + vpsubw %ymm10, %ymm8, %ymm8 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm8, 0xe0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(poly_mulcache_compute_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S new file mode 100644 index 0000000000..5f4b812684 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S @@ -0,0 +1,536 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 2) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k2_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=2) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (2 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (2 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 512 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (2 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k2_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k2_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 2) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S new file mode 100644 index 0000000000..838eb16bed --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S @@ -0,0 +1,784 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 3) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k3_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=3) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (3 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (3 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 768 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (3 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k3_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x400(%rsi), %ymm2 + vmovdqa 0x420(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x420(%rdx), %ymm5 + vmovdqa 0x200(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x440(%rsi), %ymm2 + vmovdqa 0x460(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm5 + vmovdqa 0x220(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x480(%rsi), %ymm2 + vmovdqa 0x4a0(%rsi), %ymm3 + vmovdqa 0x480(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm5 + vmovdqa 0x240(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x4c0(%rsi), %ymm2 + vmovdqa 0x4e0(%rsi), %ymm3 + vmovdqa 0x4c0(%rdx), %ymm4 + vmovdqa 0x4e0(%rdx), %ymm5 + vmovdqa 0x260(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x500(%rsi), %ymm2 + vmovdqa 0x520(%rsi), %ymm3 + vmovdqa 0x500(%rdx), %ymm4 + vmovdqa 0x520(%rdx), %ymm5 + vmovdqa 0x280(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x540(%rsi), %ymm2 + vmovdqa 0x560(%rsi), %ymm3 + vmovdqa 0x540(%rdx), %ymm4 + vmovdqa 0x560(%rdx), %ymm5 + vmovdqa 0x2a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x580(%rsi), %ymm2 + vmovdqa 0x5a0(%rsi), %ymm3 + vmovdqa 0x580(%rdx), %ymm4 + vmovdqa 0x5a0(%rdx), %ymm5 + vmovdqa 0x2c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x5c0(%rsi), %ymm2 + vmovdqa 0x5e0(%rsi), %ymm3 + vmovdqa 0x5c0(%rdx), %ymm4 + vmovdqa 0x5e0(%rdx), %ymm5 + vmovdqa 0x2e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k3_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 3) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S new file mode 100644 index 0000000000..314d938f75 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S @@ -0,0 +1,1032 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + (defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_K == 4) +/*yaml + Name: polyvec_basemul_acc_montgomery_cached_k4_avx2_asm + Description: x86_64 AVX2 base multiplication with accumulation (k=4) + Signature: void mlk_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm(int16_t *r, const int16_t *a, const int16_t *b, const int16_t *b_cache) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Output polynomial (256 x int16_t) + rsi: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t *a + description: Input polyvec a (4 x 256 x int16_t) + rdx: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const int16_t *b + description: Input polyvec b (4 x 256 x int16_t) + rcx: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int16_t *b_cache + description: Mulcache for b (4 x 128 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_polyvec_basemul_acc_montgomery_cached_k4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) +MLK_ASM_FN_SYMBOL(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0xf301f301, %eax # imm = 0xF301F301 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rsi), %ymm2 + vmovdqa 0x20(%rsi), %ymm3 + vmovdqa (%rdx), %ymm4 + vmovdqa 0x20(%rdx), %ymm5 + vmovdqa (%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x40(%rsi), %ymm2 + vmovdqa 0x60(%rsi), %ymm3 + vmovdqa 0x40(%rdx), %ymm4 + vmovdqa 0x60(%rdx), %ymm5 + vmovdqa 0x20(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x80(%rsi), %ymm2 + vmovdqa 0xa0(%rsi), %ymm3 + vmovdqa 0x80(%rdx), %ymm4 + vmovdqa 0xa0(%rdx), %ymm5 + vmovdqa 0x40(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0xc0(%rsi), %ymm2 + vmovdqa 0xe0(%rsi), %ymm3 + vmovdqa 0xc0(%rdx), %ymm4 + vmovdqa 0xe0(%rdx), %ymm5 + vmovdqa 0x60(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rsi), %ymm2 + vmovdqa 0x120(%rsi), %ymm3 + vmovdqa 0x100(%rdx), %ymm4 + vmovdqa 0x120(%rdx), %ymm5 + vmovdqa 0x80(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x140(%rsi), %ymm2 + vmovdqa 0x160(%rsi), %ymm3 + vmovdqa 0x140(%rdx), %ymm4 + vmovdqa 0x160(%rdx), %ymm5 + vmovdqa 0xa0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x180(%rsi), %ymm2 + vmovdqa 0x1a0(%rsi), %ymm3 + vmovdqa 0x180(%rdx), %ymm4 + vmovdqa 0x1a0(%rdx), %ymm5 + vmovdqa 0xc0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x1c0(%rsi), %ymm2 + vmovdqa 0x1e0(%rsi), %ymm3 + vmovdqa 0x1c0(%rdx), %ymm4 + vmovdqa 0x1e0(%rdx), %ymm5 + vmovdqa 0xe0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x200(%rsi), %ymm2 + vmovdqa 0x220(%rsi), %ymm3 + vmovdqa 0x200(%rdx), %ymm4 + vmovdqa 0x220(%rdx), %ymm5 + vmovdqa 0x100(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x240(%rsi), %ymm2 + vmovdqa 0x260(%rsi), %ymm3 + vmovdqa 0x240(%rdx), %ymm4 + vmovdqa 0x260(%rdx), %ymm5 + vmovdqa 0x120(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x280(%rsi), %ymm2 + vmovdqa 0x2a0(%rsi), %ymm3 + vmovdqa 0x280(%rdx), %ymm4 + vmovdqa 0x2a0(%rdx), %ymm5 + vmovdqa 0x140(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x2c0(%rsi), %ymm2 + vmovdqa 0x2e0(%rsi), %ymm3 + vmovdqa 0x2c0(%rdx), %ymm4 + vmovdqa 0x2e0(%rdx), %ymm5 + vmovdqa 0x160(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x300(%rsi), %ymm2 + vmovdqa 0x320(%rsi), %ymm3 + vmovdqa 0x300(%rdx), %ymm4 + vmovdqa 0x320(%rdx), %ymm5 + vmovdqa 0x180(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x340(%rsi), %ymm2 + vmovdqa 0x360(%rsi), %ymm3 + vmovdqa 0x340(%rdx), %ymm4 + vmovdqa 0x360(%rdx), %ymm5 + vmovdqa 0x1a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x380(%rsi), %ymm2 + vmovdqa 0x3a0(%rsi), %ymm3 + vmovdqa 0x380(%rdx), %ymm4 + vmovdqa 0x3a0(%rdx), %ymm5 + vmovdqa 0x1c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x3c0(%rsi), %ymm2 + vmovdqa 0x3e0(%rsi), %ymm3 + vmovdqa 0x3c0(%rdx), %ymm4 + vmovdqa 0x3e0(%rdx), %ymm5 + vmovdqa 0x1e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x400(%rsi), %ymm2 + vmovdqa 0x420(%rsi), %ymm3 + vmovdqa 0x400(%rdx), %ymm4 + vmovdqa 0x420(%rdx), %ymm5 + vmovdqa 0x200(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x440(%rsi), %ymm2 + vmovdqa 0x460(%rsi), %ymm3 + vmovdqa 0x440(%rdx), %ymm4 + vmovdqa 0x460(%rdx), %ymm5 + vmovdqa 0x220(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x480(%rsi), %ymm2 + vmovdqa 0x4a0(%rsi), %ymm3 + vmovdqa 0x480(%rdx), %ymm4 + vmovdqa 0x4a0(%rdx), %ymm5 + vmovdqa 0x240(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x4c0(%rsi), %ymm2 + vmovdqa 0x4e0(%rsi), %ymm3 + vmovdqa 0x4c0(%rdx), %ymm4 + vmovdqa 0x4e0(%rdx), %ymm5 + vmovdqa 0x260(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x500(%rsi), %ymm2 + vmovdqa 0x520(%rsi), %ymm3 + vmovdqa 0x500(%rdx), %ymm4 + vmovdqa 0x520(%rdx), %ymm5 + vmovdqa 0x280(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x540(%rsi), %ymm2 + vmovdqa 0x560(%rsi), %ymm3 + vmovdqa 0x540(%rdx), %ymm4 + vmovdqa 0x560(%rdx), %ymm5 + vmovdqa 0x2a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x580(%rsi), %ymm2 + vmovdqa 0x5a0(%rsi), %ymm3 + vmovdqa 0x580(%rdx), %ymm4 + vmovdqa 0x5a0(%rdx), %ymm5 + vmovdqa 0x2c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x5c0(%rsi), %ymm2 + vmovdqa 0x5e0(%rsi), %ymm3 + vmovdqa 0x5c0(%rdx), %ymm4 + vmovdqa 0x5e0(%rdx), %ymm5 + vmovdqa 0x2e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + vmovdqa 0x600(%rsi), %ymm2 + vmovdqa 0x620(%rsi), %ymm3 + vmovdqa 0x600(%rdx), %ymm4 + vmovdqa 0x620(%rdx), %ymm5 + vmovdqa 0x300(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa (%rdi), %ymm8 + vmovdqa 0x20(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, (%rdi) + vmovdqa %ymm9, 0x20(%rdi) + vmovdqa 0x640(%rsi), %ymm2 + vmovdqa 0x660(%rsi), %ymm3 + vmovdqa 0x640(%rdx), %ymm4 + vmovdqa 0x660(%rdx), %ymm5 + vmovdqa 0x320(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x40(%rdi), %ymm8 + vmovdqa 0x60(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x40(%rdi) + vmovdqa %ymm9, 0x60(%rdi) + vmovdqa 0x680(%rsi), %ymm2 + vmovdqa 0x6a0(%rsi), %ymm3 + vmovdqa 0x680(%rdx), %ymm4 + vmovdqa 0x6a0(%rdx), %ymm5 + vmovdqa 0x340(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x80(%rdi), %ymm8 + vmovdqa 0xa0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm9, 0xa0(%rdi) + vmovdqa 0x6c0(%rsi), %ymm2 + vmovdqa 0x6e0(%rsi), %ymm3 + vmovdqa 0x6c0(%rdx), %ymm4 + vmovdqa 0x6e0(%rdx), %ymm5 + vmovdqa 0x360(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x700(%rsi), %ymm2 + vmovdqa 0x720(%rsi), %ymm3 + vmovdqa 0x700(%rdx), %ymm4 + vmovdqa 0x720(%rdx), %ymm5 + vmovdqa 0x380(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x100(%rdi), %ymm8 + vmovdqa 0x120(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x100(%rdi) + vmovdqa %ymm9, 0x120(%rdi) + vmovdqa 0x740(%rsi), %ymm2 + vmovdqa 0x760(%rsi), %ymm3 + vmovdqa 0x740(%rdx), %ymm4 + vmovdqa 0x760(%rdx), %ymm5 + vmovdqa 0x3a0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x140(%rdi), %ymm8 + vmovdqa 0x160(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x140(%rdi) + vmovdqa %ymm9, 0x160(%rdi) + vmovdqa 0x780(%rsi), %ymm2 + vmovdqa 0x7a0(%rsi), %ymm3 + vmovdqa 0x780(%rdx), %ymm4 + vmovdqa 0x7a0(%rdx), %ymm5 + vmovdqa 0x3c0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm8, %ymm13, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x180(%rdi), %ymm8 + vmovdqa 0x1a0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm9, 0x1a0(%rdi) + vmovdqa 0x7c0(%rsi), %ymm2 + vmovdqa 0x7e0(%rsi), %ymm3 + vmovdqa 0x7c0(%rdx), %ymm4 + vmovdqa 0x7e0(%rdx), %ymm5 + vmovdqa 0x3e0(%rcx), %ymm6 + vpmullw %ymm2, %ymm1, %ymm13 + vpmullw %ymm3, %ymm1, %ymm14 + vpmullw %ymm13, %ymm4, %ymm7 + vpmullw %ymm13, %ymm5, %ymm9 + vpmullw %ymm14, %ymm6, %ymm8 + vpmullw %ymm14, %ymm4, %ymm10 + vpmulhw %ymm7, %ymm0, %ymm7 + vpmulhw %ymm9, %ymm0, %ymm9 + vpmulhw %ymm8, %ymm0, %ymm8 + vpmulhw %ymm10, %ymm0, %ymm10 + vpmulhw %ymm2, %ymm4, %ymm11 + vpmulhw %ymm2, %ymm5, %ymm12 + vpmulhw %ymm3, %ymm6, %ymm13 + vpmulhw %ymm3, %ymm4, %ymm14 + vpsubw %ymm7, %ymm11, %ymm7 + vpsubw %ymm9, %ymm12, %ymm9 + vpsubw %ymm13, %ymm8, %ymm8 + vpsubw %ymm10, %ymm14, %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm10 + vpaddw %ymm7, %ymm8, %ymm7 + vpaddw %ymm9, %ymm10, %ymm9 + vmovdqa %ymm7, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(polyvec_basemul_acc_montgomery_cached_k4_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && (MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_K == 4) */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S new file mode 100644 index 0000000000..4b5edf0b92 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_reduce_avx2_asm.S @@ -0,0 +1,234 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Kyber implementation @[REF_AVX2]. + * + * Changes: + * - Add call to csub in reduce128_avx to produce outputs + * in [0,1,...,q-1] rather than [0,1,...,q], matching the + * semantics of mlk_poly_reduce(), + * - Use a macro instead of a local function call. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: reduce_avx2_asm + Description: x86_64 AVX2 modular reduction + Signature: void mlk_reduce_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_reduce_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(reduce_avx2_asm) +MLK_ASM_FN_SYMBOL(reduce_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x4ebf4ebf, %eax # imm = 0x4EBF4EBF + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + vmovdqa (%rdi), %ymm2 + vmovdqa 0x20(%rdi), %ymm3 + vmovdqa 0x40(%rdi), %ymm4 + vmovdqa 0x60(%rdi), %ymm5 + vmovdqa 0x80(%rdi), %ymm6 + vmovdqa 0xa0(%rdi), %ymm7 + vmovdqa 0xc0(%rdi), %ymm8 + vmovdqa 0xe0(%rdi), %ymm9 + vpmulhw %ymm1, %ymm2, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm2, %ymm2 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vpmulhw %ymm1, %ymm4, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmulhw %ymm1, %ymm5, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmulhw %ymm1, %ymm6, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vpmulhw %ymm1, %ymm8, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpsubw %ymm0, %ymm2, %ymm2 + vpsraw $0xf, %ymm2, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm2, %ymm2 + vpsubw %ymm0, %ymm3, %ymm3 + vpsraw $0xf, %ymm3, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm3, %ymm3 + vpsubw %ymm0, %ymm4, %ymm4 + vpsraw $0xf, %ymm4, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm4, %ymm4 + vpsubw %ymm0, %ymm5, %ymm5 + vpsraw $0xf, %ymm5, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm0, %ymm6, %ymm6 + vpsraw $0xf, %ymm6, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm6, %ymm6 + vpsubw %ymm0, %ymm7, %ymm7 + vpsraw $0xf, %ymm7, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm0, %ymm8, %ymm8 + vpsraw $0xf, %ymm8, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm0, %ymm9, %ymm9 + vpsraw $0xf, %ymm9, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm9, %ymm9 + vmovdqa %ymm2, (%rdi) + vmovdqa %ymm3, 0x20(%rdi) + vmovdqa %ymm4, 0x40(%rdi) + vmovdqa %ymm5, 0x60(%rdi) + vmovdqa %ymm6, 0x80(%rdi) + vmovdqa %ymm7, 0xa0(%rdi) + vmovdqa %ymm8, 0xc0(%rdi) + vmovdqa %ymm9, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm2 + vmovdqa 0x120(%rdi), %ymm3 + vmovdqa 0x140(%rdi), %ymm4 + vmovdqa 0x160(%rdi), %ymm5 + vmovdqa 0x180(%rdi), %ymm6 + vmovdqa 0x1a0(%rdi), %ymm7 + vmovdqa 0x1c0(%rdi), %ymm8 + vmovdqa 0x1e0(%rdi), %ymm9 + vpmulhw %ymm1, %ymm2, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm2, %ymm2 + vpmulhw %ymm1, %ymm3, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm3, %ymm3 + vpmulhw %ymm1, %ymm4, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmulhw %ymm1, %ymm5, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm5, %ymm5 + vpmulhw %ymm1, %ymm6, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm6, %ymm6 + vpmulhw %ymm1, %ymm7, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm7, %ymm7 + vpmulhw %ymm1, %ymm8, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm8, %ymm8 + vpmulhw %ymm1, %ymm9, %ymm12 + vpsraw $0xa, %ymm12, %ymm12 + vpmullw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpsubw %ymm0, %ymm2, %ymm2 + vpsraw $0xf, %ymm2, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm2, %ymm2 + vpsubw %ymm0, %ymm3, %ymm3 + vpsraw $0xf, %ymm3, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm3, %ymm3 + vpsubw %ymm0, %ymm4, %ymm4 + vpsraw $0xf, %ymm4, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm4, %ymm4 + vpsubw %ymm0, %ymm5, %ymm5 + vpsraw $0xf, %ymm5, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm5, %ymm5 + vpsubw %ymm0, %ymm6, %ymm6 + vpsraw $0xf, %ymm6, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm6, %ymm6 + vpsubw %ymm0, %ymm7, %ymm7 + vpsraw $0xf, %ymm7, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm7, %ymm7 + vpsubw %ymm0, %ymm8, %ymm8 + vpsraw $0xf, %ymm8, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm8, %ymm8 + vpsubw %ymm0, %ymm9, %ymm9 + vpsraw $0xf, %ymm9, %ymm12 + vpand %ymm0, %ymm12, %ymm12 + vpaddw %ymm12, %ymm9, %ymm9 + vmovdqa %ymm2, 0x100(%rdi) + vmovdqa %ymm3, 0x120(%rdi) + vmovdqa %ymm4, 0x140(%rdi) + vmovdqa %ymm5, 0x160(%rdi) + vmovdqa %ymm6, 0x180(%rdi) + vmovdqa %ymm7, 0x1a0(%rdi) + vmovdqa %ymm8, 0x1c0(%rdi) + vmovdqa %ymm9, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(reduce_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S new file mode 100644 index 0000000000..789c876e1a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_rej_uniform_avx2_asm.S @@ -0,0 +1,136 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/************************************************* + * Name: mlk_rej_uniform_avx2_asm + * + * Description: Run rejection sampling on uniform random bytes to generate + * uniform random integers mod q + * + * Arguments: - int16_t *r: pointer to output buffer of MLKEM_N + * 16-bit coefficients. + * - const uint8_t *buf: pointer to input buffer + * (assumed to be uniform random bytes) + * - unsigned buflen: length of input buffer in bytes. + * Must be a multiple of 12. + * + * Returns number of sampled 16-bit integers (at most MLKEM_N). + **************************************************/ +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/*yaml + Name: rej_uniform_avx2_asm + Description: x86_64 AVX2 rejection sampling + Signature: uint64_t mlk_rej_uniform_avx2_asm(int16_t *r, const uint8_t *buf, unsigned buflen, const uint8_t *table) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: write-only + c_parameter: int16_t *r + description: Output buffer (256 x int16_t) + rsi: + type: buffer + size_bytes: rdx + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + rdx: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be multiple of 12) + test_with: 504 # MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE + rcx: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t *table + description: Lookup table +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_rej_uniform_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(rej_uniform_avx2_asm) +MLK_ASM_FN_SYMBOL(rej_uniform_avx2_asm) + + .cfi_startproc + subq $0x210, %rsp # imm = 0x210 + .cfi_adjust_cfa_offset 0x210 + xorl %eax, %eax + testq %rdx, %rdx + je Lmlk_rej_uniform_asm_end + movabsq $0xd010d010d010d01, %rax # imm = 0xD010D010D010D01 + movq %rax, %xmm0 + pinsrq $0x1, %rax, %xmm0 + movabsq $0xfff0fff0fff0fff, %rax # imm = 0xFFF0FFF0FFF0FFF + movq %rax, %xmm5 + pinsrq $0x1, %rax, %xmm5 + movabsq $0x504040302010100, %rax # imm = 0x504040302010100 + movq %rax, %xmm4 + movabsq $0xb0a0a0908070706, %rax # imm = 0xB0A0A0908070706 + pinsrq $0x1, %rax, %xmm4 + movq $0x0, %rax + movq $0x0, %r8 + movq $0x5555, %r9 # imm = 0x5555 + +Lmlk_rej_uniform_asm_loop_start: + movq (%rsi,%r8), %xmm2 + pinsrd $0x2, 0x8(%rsi,%r8), %xmm2 + pshufb %xmm4, %xmm2 + movdqa %xmm2, %xmm3 + psrlw $0x4, %xmm3 + pblendw $0xaa, %xmm3, %xmm2 # xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7] + pand %xmm5, %xmm2 + movdqa %xmm0, %xmm1 + pcmpgtw %xmm2, %xmm1 + pmovmskb %xmm1, %r11d + pextq %r9, %r11, %r11 + movq %r11, %r10 + shlq $0x4, %r10 + movdqu (%rcx,%r10), %xmm3 + pshufb %xmm3, %xmm2 + movdqu %xmm2, (%rsp,%rax,2) + popcntq %r11, %r11 + addq %r11, %rax + cmpq $0x100, %rax # imm = 0x100 + jae Lmlk_rej_uniform_asm_final_copy + addq $0xc, %r8 + cmpq %r8, %rdx + ja Lmlk_rej_uniform_asm_loop_start + +Lmlk_rej_uniform_asm_final_copy: + movq $0x100, %rcx # imm = 0x100 + cmpq $0x100, %rax # imm = 0x100 + cmovaq %rcx, %rax + movq %rsp, %rsi + movq %rax, %rcx + shlq %rcx + rep movsb (%rsi), %es:(%rdi) + +Lmlk_rej_uniform_asm_end: + addq $0x210, %rsp # imm = 0x210 + .cfi_adjust_cfa_offset -0x210 + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(rej_uniform_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S new file mode 100644 index 0000000000..cf6268ba01 --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/mlkem_tomont_avx2_asm.S @@ -0,0 +1,172 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Kyber optimized AVX2 implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/avx2 + */ + +/* + * Implementation from Kyber reference repository @[REF_AVX2] + * + * Changes: + * - Add call to csub in reduce128_avx to produce outputs + * in [0,1,...,q-1] rather than [0,1,...,q], matching the + * semantics of mlk_poly_reduce(), + * - Use a macro instead of a local function call. + */ + +#include "../../../common.h" +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) && \ + !defined(MLK_CONFIG_NO_KEYPAIR_API) +/*yaml + Name: tomont_avx2_asm + Description: x86_64 AVX2 Montgomery conversion + Signature: void mlk_tomont_avx2_asm(int16_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 512 + permissions: read/write + c_parameter: int16_t *r + description: Input/output polynomial (256 x int16_t) +*/ + + +/* + * WARNING: This file is auto-derived from the mlkem-native source file + * dev/x86_64/src/mlkem_tomont_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 +.global MLK_ASM_NAMESPACE(tomont_avx2_asm) +MLK_ASM_FN_SYMBOL(tomont_avx2_asm) + + .cfi_startproc + movl $0xd010d01, %eax # imm = 0xD010D01 + vmovd %eax, %xmm0 + vpbroadcastd %xmm0, %ymm0 + movl $0x50495049, %eax # imm = 0x50495049 + vmovd %eax, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x5490549, %eax # imm = 0x5490549 + vmovd %eax, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vmovdqa (%rdi), %ymm3 + vmovdqa 0x20(%rdi), %ymm4 + vmovdqa 0x40(%rdi), %ymm5 + vmovdqa 0x60(%rdi), %ymm6 + vmovdqa 0x80(%rdi), %ymm7 + vmovdqa 0xa0(%rdi), %ymm8 + vmovdqa 0xc0(%rdi), %ymm9 + vmovdqa 0xe0(%rdi), %ymm10 + vpmullw %ymm1, %ymm3, %ymm11 + vpmulhw %ymm2, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm3, %ymm3 + vpmullw %ymm1, %ymm4, %ymm12 + vpmulhw %ymm2, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm1, %ymm5, %ymm13 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm5, %ymm5 + vpmullw %ymm1, %ymm6, %ymm14 + vpmulhw %ymm2, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm14, %ymm14 + vpsubw %ymm14, %ymm6, %ymm6 + vpmullw %ymm1, %ymm7, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm15, %ymm15 + vpsubw %ymm15, %ymm7, %ymm7 + vpmullw %ymm1, %ymm8, %ymm11 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm8, %ymm8 + vpmullw %ymm1, %ymm9, %ymm12 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm1, %ymm10, %ymm13 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm10, %ymm10 + vmovdqa %ymm3, (%rdi) + vmovdqa %ymm4, 0x20(%rdi) + vmovdqa %ymm5, 0x40(%rdi) + vmovdqa %ymm6, 0x60(%rdi) + vmovdqa %ymm7, 0x80(%rdi) + vmovdqa %ymm8, 0xa0(%rdi) + vmovdqa %ymm9, 0xc0(%rdi) + vmovdqa %ymm10, 0xe0(%rdi) + vmovdqa 0x100(%rdi), %ymm3 + vmovdqa 0x120(%rdi), %ymm4 + vmovdqa 0x140(%rdi), %ymm5 + vmovdqa 0x160(%rdi), %ymm6 + vmovdqa 0x180(%rdi), %ymm7 + vmovdqa 0x1a0(%rdi), %ymm8 + vmovdqa 0x1c0(%rdi), %ymm9 + vmovdqa 0x1e0(%rdi), %ymm10 + vpmullw %ymm1, %ymm3, %ymm11 + vpmulhw %ymm2, %ymm3, %ymm3 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm3, %ymm3 + vpmullw %ymm1, %ymm4, %ymm12 + vpmulhw %ymm2, %ymm4, %ymm4 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm4, %ymm4 + vpmullw %ymm1, %ymm5, %ymm13 + vpmulhw %ymm2, %ymm5, %ymm5 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm5, %ymm5 + vpmullw %ymm1, %ymm6, %ymm14 + vpmulhw %ymm2, %ymm6, %ymm6 + vpmulhw %ymm0, %ymm14, %ymm14 + vpsubw %ymm14, %ymm6, %ymm6 + vpmullw %ymm1, %ymm7, %ymm15 + vpmulhw %ymm2, %ymm7, %ymm7 + vpmulhw %ymm0, %ymm15, %ymm15 + vpsubw %ymm15, %ymm7, %ymm7 + vpmullw %ymm1, %ymm8, %ymm11 + vpmulhw %ymm2, %ymm8, %ymm8 + vpmulhw %ymm0, %ymm11, %ymm11 + vpsubw %ymm11, %ymm8, %ymm8 + vpmullw %ymm1, %ymm9, %ymm12 + vpmulhw %ymm2, %ymm9, %ymm9 + vpmulhw %ymm0, %ymm12, %ymm12 + vpsubw %ymm12, %ymm9, %ymm9 + vpmullw %ymm1, %ymm10, %ymm13 + vpmulhw %ymm2, %ymm10, %ymm10 + vpmulhw %ymm0, %ymm13, %ymm13 + vpsubw %ymm13, %ymm10, %ymm10 + vmovdqa %ymm3, 0x100(%rdi) + vmovdqa %ymm4, 0x120(%rdi) + vmovdqa %ymm5, 0x140(%rdi) + vmovdqa %ymm6, 0x160(%rdi) + vmovdqa %ymm7, 0x180(%rdi) + vmovdqa %ymm8, 0x1a0(%rdi) + vmovdqa %ymm9, 0x1c0(%rdi) + vmovdqa %ymm10, 0x1e0(%rdi) + retq + .cfi_endproc + +MLK_ASM_FN_SIZE(tomont_avx2_asm) + +#endif /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + && !MLK_CONFIG_NO_KEYPAIR_API */ + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c b/src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c new file mode 100644 index 0000000000..d62256af5a --- /dev/null +++ b/src/mlkem-native/mlkem/src/native/x86_64/src/rej_uniform_table.c @@ -0,0 +1,545 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLK_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_x86_64.h" + +/* + * Lookup table used by rejection sampling of the public matrix. + * See autogen for details. + */ +MLK_ALIGN MLK_INTERNAL_DATA_DEFINITION const uint8_t + mlk_rej_uniform_table[4096] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 0 */, + 0, 1, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 1 */, + 2, 3, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 2 */, + 0, 1, 2, 3, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 3 */, + 4, 5, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 4 */, + 0, 1, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 5 */, + 2, 3, 4, 5, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 6 */, + 0, 1, 2, 3, 4, 5, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 7 */, + 6, 7, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 8 */, + 0, 1, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 9 */, + 2, 3, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 10 */, + 0, 1, 2, 3, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 11 */, + 4, 5, 6, 7, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 12 */, + 0, 1, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 13 */, + 2, 3, 4, 5, 6, 7, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 14 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 255, 255, 255, 255, 255, 255, 255, 255 /* 15 */, + 8, 9, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 16 */, + 0, 1, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 17 */, + 2, 3, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 18 */, + 0, 1, 2, 3, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 19 */, + 4, 5, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 20 */, + 0, 1, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 21 */, + 2, 3, 4, 5, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 22 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 23 */, + 6, 7, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 24 */, + 0, 1, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 25 */, + 2, 3, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 26 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 27 */, + 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 28 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 29 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 255, 255, 255, 255, 255, 255, 255, 255 /* 30 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 255, 255, 255, 255, 255, 255 /* 31 */, + 10, 11, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 32 */, + 0, 1, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 33 */, + 2, 3, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 34 */, + 0, 1, 2, 3, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 35 */, + 4, 5, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 36 */, + 0, 1, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 37 */, + 2, 3, 4, 5, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 38 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 39 */, + 6, 7, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 40 */, + 0, 1, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 41 */, + 2, 3, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 42 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 43 */, + 4, 5, 6, 7, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 44 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 45 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 46 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 255, 255, 255, 255, 255, 255 /* 47 */, + 8, 9, 10, 11, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 48 */, + 0, 1, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 49 */, + 2, 3, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 50 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 51 */, + 4, 5, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 52 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 53 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 54 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 55 */, + 6, 7, 8, 9, 10, 11, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 56 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 57 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 58 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 59 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 255, 255, 255, 255, 255, 255, 255, 255 /* 60 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 61 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 255, 255, 255, 255, 255, 255 /* 62 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 255, 255, 255, 255 /* 63 */, + 12, 13, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 64 */, + 0, 1, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 65 */, + 2, 3, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 66 */, + 0, 1, 2, 3, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 67 */, + 4, 5, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 68 */, + 0, 1, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 69 */, + 2, 3, 4, 5, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 70 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 71 */, + 6, 7, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 72 */, + 0, 1, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 73 */, + 2, 3, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 74 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 75 */, + 4, 5, 6, 7, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 76 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 77 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 78 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 255, 255, 255, 255, 255, 255 /* 79 */, + 8, 9, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 80 */, + 0, 1, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 81 */, + 2, 3, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 82 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 83 */, + 4, 5, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 84 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 85 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 86 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 87 */, + 6, 7, 8, 9, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 88 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 89 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 90 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 91 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 92 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 93 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 255, 255, 255, 255, 255, 255 /* 94 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 255, 255, 255, 255 /* 95 */, + 10, 11, 12, 13, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 96 */, + 0, 1, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 97 */, + 2, 3, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 98 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 99 */, + 4, 5, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 100 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 101 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 102 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 103 */, + 6, 7, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 104 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 105 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 106 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 107 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 108 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 109 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 110 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 255, 255, 255, 255 /* 111 */, + 8, 9, 10, 11, 12, 13, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 112 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 113 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 114 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 115 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 116 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 117 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 118 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 119 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 255, 255, 255, 255, 255, 255, 255, 255 /* 120 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 121 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 122 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 123 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 255, 255, 255, 255, 255, 255 /* 124 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 125 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 255, 255, 255, 255 /* 126 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 255, 255 /* 127 */, + 14, 15, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 128 */, + 0, 1, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 129 */, + 2, 3, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 130 */, + 0, 1, 2, 3, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 131 */, + 4, 5, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 132 */, + 0, 1, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 133 */, + 2, 3, 4, 5, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 134 */, + 0, 1, 2, 3, 4, 5, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 135 */, + 6, 7, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 136 */, + 0, 1, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 137 */, + 2, 3, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 138 */, + 0, 1, 2, 3, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 139 */, + 4, 5, 6, 7, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 140 */, + 0, 1, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 141 */, + 2, 3, 4, 5, 6, 7, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 142 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 14, 15, 255, 255, 255, 255, 255, 255 /* 143 */, + 8, 9, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 144 */, + 0, 1, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 145 */, + 2, 3, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 146 */, + 0, 1, 2, 3, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 147 */, + 4, 5, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 148 */, + 0, 1, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 149 */, + 2, 3, 4, 5, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 150 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 151 */, + 6, 7, 8, 9, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 152 */, + 0, 1, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 153 */, + 2, 3, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 154 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 155 */, + 4, 5, 6, 7, 8, 9, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 156 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 157 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 14, 15, 255, 255, 255, 255, 255, 255 /* 158 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 14, 15, 255, 255, 255, 255 /* 159 */, + 10, 11, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 160 */, + 0, 1, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 161 */, + 2, 3, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 162 */, + 0, 1, 2, 3, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 163 */, + 4, 5, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 164 */, + 0, 1, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 165 */, + 2, 3, 4, 5, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 166 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 167 */, + 6, 7, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 168 */, + 0, 1, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 169 */, + 2, 3, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 170 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 171 */, + 4, 5, 6, 7, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 172 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 173 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 174 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 14, 15, 255, 255, 255, 255 /* 175 */, + 8, 9, 10, 11, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 176 */, + 0, 1, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 177 */, + 2, 3, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 178 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 179 */, + 4, 5, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 180 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 181 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 182 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 183 */, + 6, 7, 8, 9, 10, 11, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 184 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 185 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 186 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 187 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 14, 15, 255, 255, 255, 255, 255, 255 /* 188 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 189 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 255, 255, 255, 255 /* 190 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 14, 15, 255, 255 /* 191 */, + 12, 13, 14, 15, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 192 */, + 0, 1, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 193 */, + 2, 3, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 194 */, + 0, 1, 2, 3, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 195 */, + 4, 5, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 196 */, + 0, 1, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 197 */, + 2, 3, 4, 5, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 198 */, + 0, 1, 2, 3, 4, 5, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 199 */, + 6, 7, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 200 */, + 0, 1, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 201 */, + 2, 3, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 202 */, + 0, 1, 2, 3, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 203 */, + 4, 5, 6, 7, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 204 */, + 0, 1, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 205 */, + 2, 3, 4, 5, 6, 7, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 206 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 14, 15, 255, 255, 255, 255 /* 207 */, + 8, 9, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 208 */, + 0, 1, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 209 */, + 2, 3, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 210 */, + 0, 1, 2, 3, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 211 */, + 4, 5, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 212 */, + 0, 1, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 213 */, + 2, 3, 4, 5, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 214 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 215 */, + 6, 7, 8, 9, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 216 */, + 0, 1, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 217 */, + 2, 3, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 218 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 219 */, + 4, 5, 6, 7, 8, 9, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 220 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 221 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 14, 15, 255, 255, 255, 255 /* 222 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 14, 15, 255, 255 /* 223 */, + 10, 11, 12, 13, 14, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255 /* 224 */, + 0, 1, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 225 */, + 2, 3, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 226 */, + 0, 1, 2, 3, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 227 */, + 4, 5, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 228 */, + 0, 1, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 229 */, + 2, 3, 4, 5, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 230 */, + 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 231 */, + 6, 7, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 232 */, + 0, 1, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 233 */, + 2, 3, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 234 */, + 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 235 */, + 4, 5, 6, 7, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 236 */, + 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 237 */, + 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 238 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 12, 13, 14, 15, 255, 255 /* 239 */, + 8, 9, 10, 11, 12, 13, 14, 15, + 255, 255, 255, 255, 255, 255, 255, 255 /* 240 */, + 0, 1, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 241 */, + 2, 3, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 242 */, + 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 243 */, + 4, 5, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 244 */, + 0, 1, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 245 */, + 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 246 */, + 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 247 */, + 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 255, 255, 255, 255, 255, 255 /* 248 */, + 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 249 */, + 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 250 */, + 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 251 */, + 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 255, 255, 255, 255 /* 252 */, + 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 253 */, + 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 255, 255 /* 254 */, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 /* 255 */, +}; + +#else /* MLK_ARITH_BACKEND_X86_64_DEFAULT && !MLK_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLK_EMPTY_CU(avx2_rej_uniform_table) + +#endif /* !(MLK_ARITH_BACKEND_X86_64_DEFAULT && \ + !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/params.h b/src/mlkem-native/mlkem/src/params.h new file mode 100644 index 0000000000..04598539c4 --- /dev/null +++ b/src/mlkem-native/mlkem/src/params.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_PARAMS_H +#define MLK_PARAMS_H + +#if !defined(MLK_CONFIG_PARAMETER_SET) +#error MLK_CONFIG_PARAMETER_SET is not defined +#endif + +#if MLK_CONFIG_PARAMETER_SET == 512 +#define MLKEM_K 2 +#elif MLK_CONFIG_PARAMETER_SET == 768 +#define MLKEM_K 3 +#elif MLK_CONFIG_PARAMETER_SET == 1024 +#define MLKEM_K 4 +#else +#error Invalid value for MLK_CONFIG_PARAMETER_SET. Must be 512, 768, or 1024. +#endif + +#define MLKEM_N 256 +#define MLKEM_Q 3329 +#define MLKEM_Q_HALF ((MLKEM_Q + 1) / 2) /* 1665 */ +#define MLKEM_UINT12_LIMIT 4096 + +#define MLKEM_SYMBYTES 32 /* size in bytes of hashes, and seeds */ +#define MLKEM_SSBYTES 32 /* size in bytes of shared key */ + +#define MLKEM_POLYBYTES 384 +#define MLKEM_POLYVECBYTES (MLKEM_K * MLKEM_POLYBYTES) + +#define MLKEM_POLYCOMPRESSEDBYTES_D4 128 +#define MLKEM_POLYCOMPRESSEDBYTES_D5 160 +#define MLKEM_POLYCOMPRESSEDBYTES_D10 320 +#define MLKEM_POLYCOMPRESSEDBYTES_D11 352 + +#if MLKEM_K == 2 +#define MLKEM_ETA1 3 +#define MLKEM_DU 10 +#define MLKEM_DV 4 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D4 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D10 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#elif MLKEM_K == 3 +#define MLKEM_ETA1 2 +#define MLKEM_DU 10 +#define MLKEM_DV 4 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D4 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D10 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#elif MLKEM_K == 4 +#define MLKEM_ETA1 2 +#define MLKEM_DU 11 +#define MLKEM_DV 5 +#define MLKEM_POLYCOMPRESSEDBYTES_DV MLKEM_POLYCOMPRESSEDBYTES_D5 +#define MLKEM_POLYCOMPRESSEDBYTES_DU MLKEM_POLYCOMPRESSEDBYTES_D11 +#define MLKEM_POLYVECCOMPRESSEDBYTES_DU (MLKEM_K * MLKEM_POLYCOMPRESSEDBYTES_DU) +#endif /* MLKEM_K == 4 */ + +#define MLKEM_ETA2 2 + +#define MLKEM_INDCPA_MSGBYTES (MLKEM_SYMBYTES) +#define MLKEM_INDCPA_PUBLICKEYBYTES (MLKEM_POLYVECBYTES + MLKEM_SYMBYTES) +#define MLKEM_INDCPA_SECRETKEYBYTES (MLKEM_POLYVECBYTES) +#define MLKEM_INDCPA_BYTES \ + (MLKEM_POLYVECCOMPRESSEDBYTES_DU + MLKEM_POLYCOMPRESSEDBYTES_DV) + +#define MLKEM_INDCCA_PUBLICKEYBYTES (MLKEM_INDCPA_PUBLICKEYBYTES) +/* 32 bytes of additional space to save H(pk) */ +#define MLKEM_INDCCA_SECRETKEYBYTES \ + (MLKEM_INDCPA_SECRETKEYBYTES + MLKEM_INDCPA_PUBLICKEYBYTES + \ + 2 * MLKEM_SYMBYTES) +#define MLKEM_INDCCA_CIPHERTEXTBYTES (MLKEM_INDCPA_BYTES) + +#endif /* !MLK_PARAMS_H */ diff --git a/src/mlkem-native/mlkem/src/poly.c b/src/mlkem-native/mlkem/src/poly.c new file mode 100644 index 0000000000..edc17a2b18 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly.c @@ -0,0 +1,581 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + + +#include "cbmc.h" +#include "debug.h" +#include "poly.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/** + * Montgomery multiplication modulo MLKEM_Q. + * + * @reference{`fqmul()` in the reference implementation @[REF].} + * + * @param a First factor. Can be any int16_t. + * @param b Second factor. Must be signed canonical + * (abs value < (MLKEM_Q+1)/2). + * + * @return 16-bit integer congruent to a*b*R^{-1} mod MLKEM_Q, and + * smaller than MLKEM_Q in absolute value. + */ +static MLK_INLINE int16_t mlk_fqmul(int16_t a, int16_t b) +__contract__( + requires(b > -MLKEM_Q_HALF && b < MLKEM_Q_HALF) + ensures(return_value > -MLKEM_Q && return_value < MLKEM_Q) +) +{ + int16_t res; + mlk_assert_abs_bound(&b, 1, MLKEM_Q_HALF); + + res = mlk_montgomery_reduce((int32_t)a * (int32_t)b); + /* Bounds: + * |res| <= ceil(|a| * |b| / 2^16) + (MLKEM_Q + 1) / 2 + * <= ceil(2^15 * ((MLKEM_Q - 1)/2) / 2^16) + (MLKEM_Q + 1) / 2 + * <= ceil((MLKEM_Q - 1) / 4) + (MLKEM_Q + 1) / 2 + * < MLKEM_Q + */ + + mlk_assert_abs_bound(&res, 1, MLKEM_Q); + return res; +} + +/** + * Barrett reduction; given a 16-bit integer a, computes the centered + * representative congruent to a mod MLKEM_Q in [-(MLKEM_Q-1)/2, (MLKEM_Q-1)/2]. + * + * @reference{`barrett_reduce()` in the reference implementation @[REF].} + * + * @param a Input integer to be reduced. + * + * @return Integer in [-(MLKEM_Q-1)/2, (MLKEM_Q-1)/2] congruent to @p a modulo + * MLKEM_Q. + */ +static MLK_INLINE int16_t mlk_barrett_reduce(int16_t a) +__contract__( + ensures(return_value > -MLKEM_Q_HALF && return_value < MLKEM_Q_HALF) +) +{ + /* Barrett reduction approximates + * ``` + * round(a/MLKEM_Q) + * = round(a*(2^N/MLKEM_Q))/2^N) + * ~= round(a*round(2^N/MLKEM_Q)/2^N) + * ``` + * Here, we pick N=26. + */ + const int32_t magic = 20159; /* check-magic: 20159 == round(2^26 / MLKEM_Q) */ + + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + const int32_t t = (magic * a + ((int32_t)1 << 25)) >> 26; + + /* + * t is in -10 .. +10, so we need 32-bit math to + * evaluate t * MLKEM_Q and the subsequent subtraction + */ + int16_t res = (int16_t)(a - t * MLKEM_Q); + + mlk_assert_abs_bound(&res, 1, MLKEM_Q_HALF); + return res; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/* Reference: `poly_tomont()` in the reference implementation @[REF]. */ +MLK_STATIC_TESTABLE void mlk_poly_tomont_c(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) +) +{ + unsigned i; + const int16_t f = 1353; /* check-magic: 1353 == signed_mod(2^32, MLKEM_Q) */ + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(array_abs_bound(r->coeffs, 0, i, MLKEM_Q)) + decreases(MLKEM_N - i)) + { + r->coeffs[i] = mlk_fqmul(r->coeffs[i], f); + } + + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_tomont(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_POLY_TOMONT) + int ret; + ret = mlk_poly_tomont_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_TOMONT */ + + mlk_poly_tomont_c(r); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +/** + * Constant-time conversion of signed representatives modulo MLKEM_Q within + * range [-(MLKEM_Q-1), MLKEM_Q-1] into unsigned representatives within + * range [0, MLKEM_Q-1]. + * + * @reference{Not present in the reference implementation @[REF]. Used here + * to implement different semantics of `poly_reduce()`; see below. In the + * reference implementation @[REF] this logic is part of all compression + * functions (see `compress.c`).} + * + * @param c Signed coefficient to be converted. + * + * @return Unsigned representative in [0, MLKEM_Q). + */ +static MLK_INLINE int16_t mlk_scalar_signed_to_unsigned_q(int16_t c) +__contract__( + requires(c > -MLKEM_Q && c < MLKEM_Q) + ensures(return_value >= 0 && return_value < MLKEM_Q) + ensures(return_value == (int32_t)c + (((int32_t)c < 0) * MLKEM_Q))) +{ + mlk_assert_abs_bound(&c, 1, MLKEM_Q); + + /* Add MLKEM_Q if c is negative, but in constant time. + * + * Note that c + MLKEM_Q does not overflow in int16_t, + * so the cast to uint16_t is safe. */ + c = mlk_ct_sel_int16((int16_t)(c + MLKEM_Q), c, mlk_ct_cmask_neg_i16(c)); + + mlk_assert_bound(&c, 1, 0, MLKEM_Q); + return c; +} + +/* Reference: `poly_reduce()` in the reference implementation @[REF] + * - We use _unsigned_ canonical outputs, while the reference + * implementation uses _signed_ canonical outputs. + * Accordingly, we need a conditional addition of MLKEM_Q + * here to go from signed to unsigned representatives. + * This conditional addition is then dropped from all + * polynomial compression functions instead (see `compress.c`). */ +MLK_STATIC_TESTABLE void mlk_poly_reduce_c(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +) +{ + unsigned i; + + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(array_bound(r->coeffs, 0, i, 0, MLKEM_Q)) + decreases(MLKEM_N - i)) + { + /* Barrett reduction, giving signed canonical representative */ + int16_t t = mlk_barrett_reduce(r->coeffs[i]); + /* Conditional addition to get unsigned canonical representative */ + r->coeffs[i] = mlk_scalar_signed_to_unsigned_q(t); + } + + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_reduce(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_POLY_REDUCE) + int ret; + ret = mlk_poly_reduce_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_bound(r, MLKEM_N, 0, MLKEM_Q); + return; + } +#endif /* MLK_USE_NATIVE_POLY_REDUCE */ + + mlk_poly_reduce_c(r); +} + +/* Reference: `poly_add()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_poly_add(mlk_poly *r, const mlk_poly *b) +{ + unsigned i; + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(forall(k0, i, MLKEM_N, r->coeffs[k0] == loop_entry(*r).coeffs[k0])) + invariant(forall(k1, 0, i, r->coeffs[k1] == loop_entry(*r).coeffs[k1] + b->coeffs[k1])) + decreases(MLKEM_N - i)) + { + /* The preconditions imply that the addition stays within int16_t. */ + r->coeffs[i] = (int16_t)(r->coeffs[i] + b->coeffs[i]); + } +} + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `poly_sub()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_poly_sub(mlk_poly *r, const mlk_poly *b) +{ + unsigned i; + for (i = 0; i < MLKEM_N; i++) + __loop__( + invariant(i <= MLKEM_N) + invariant(forall(k0, i, MLKEM_N, r->coeffs[k0] == loop_entry(*r).coeffs[k0])) + invariant(forall(k1, 0, i, r->coeffs[k1] == loop_entry(*r).coeffs[k1] - b->coeffs[k1])) + decreases(MLKEM_N - i)) + { + /* The preconditions imply that the subtraction stays within int16_t. */ + r->coeffs[i] = (int16_t)(r->coeffs[i] - b->coeffs[i]); + } +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#include "zetas.inc" + +/* Reference: Does not exist in the reference implementation @[REF]. + * - The reference implementation does not use a + * multiplication cache ('mulcache'). This idea originates + * from @[NeonNTT] and is used at the C level here. */ +MLK_STATIC_TESTABLE void mlk_poly_mulcache_compute_c(mlk_poly_mulcache *x, + const mlk_poly *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_poly_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_poly))) + assigns(memory_slice(x, sizeof(mlk_poly_mulcache))) +) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 4; i++) + __loop__( + invariant(i <= MLKEM_N / 4) + invariant(array_abs_bound(x->coeffs, 0, 2 * i, MLKEM_Q)) + decreases(MLKEM_N / 4 - i)) + { + x->coeffs[2 * i + 0] = mlk_fqmul(a->coeffs[4 * i + 1], mlk_zetas[64 + i]); + /* The values in zeta table are <= MLKEM_Q in absolute value, + * so the negation in int16_t is safe. */ + x->coeffs[2 * i + 1] = + mlk_fqmul(a->coeffs[4 * i + 3], (int16_t)(-mlk_zetas[64 + i])); + } + + /* + * This bound is true for the C implementation, but not needed + * in the higher level bounds reasoning. It is thus omitted + * from the spec to not unnecessarily constrain native + * implementations, but checked here nonetheless. + */ + mlk_assert_abs_bound(x, MLKEM_N / 2, MLKEM_Q); +} + +MLK_INTERNAL_API +void mlk_poly_mulcache_compute(mlk_poly_mulcache *x, const mlk_poly *a) +{ +#if defined(MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE) + int ret; + ret = mlk_poly_mulcache_compute_native(x->coeffs, a->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } +#endif /* MLK_USE_NATIVE_POLY_MULCACHE_COMPUTE */ + + mlk_poly_mulcache_compute_c(x, a); +} + +/* + * Computes a block CT butterflies with a fixed twiddle factor, + * using Montgomery multiplication. + * Parameters: + * - r: Pointer to base of polynomial (_not_ the base of butterfly block) + * - root: Twiddle factor to use for the butterfly. This must be in + * Montgomery form and signed canonical. + * - start: Offset to the beginning of the butterfly block + * - len: Index difference between coefficients subject to a butterfly + * - bound: Ghost variable describing coefficient bound: Prior to `start`, + * coefficients must be bound by `bound + MLKEM_Q`. Post `start`, + * they must be bound by `bound`. + * When this function returns, output coefficients in the index range + * [start, start+2*len) have bound bumped to `bound + MLKEM_Q`. + * Example: + * - start=8, len=4 + * This would compute the following four butterflies + * 8 -- 12 + * 9 -- 13 + * 10 -- 14 + * 11 -- 15 + * - start=4, len=2 + * This would compute the following two butterflies + * 4 -- 6 + * 5 -- 7 + */ + +/* Reference: Embedded in `ntt()` in the reference implementation @[REF]. */ +static void mlk_ntt_butterfly_block(int16_t r[MLKEM_N], int16_t zeta, + unsigned start, unsigned len, + unsigned bound) +__contract__( + requires(start < MLKEM_N) + requires(1 <= len && len <= MLKEM_N / 2 && start + 2 * len <= MLKEM_N) + requires(0 <= bound && bound < INT16_MAX - MLKEM_Q) + requires(-MLKEM_Q_HALF < zeta && zeta < MLKEM_Q_HALF) + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(array_abs_bound(r, 0, start, bound + MLKEM_Q)) + requires(array_abs_bound(r, start, MLKEM_N, bound)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, start + 2*len, bound + MLKEM_Q)) + ensures(array_abs_bound(r, start + 2 * len, MLKEM_N, bound))) +{ + /* `bound` is a ghost variable only needed in the CBMC specification */ + unsigned j; + ((void)bound); + for (j = start; j < start + len; j++) + __loop__( + invariant(start <= j && j <= start + len) + /* + * Coefficients are updated in strided pairs, so the bounds for the + * intermediate states alternate twice between the old and new bound + */ + invariant(array_abs_bound(r, 0, j, bound + MLKEM_Q)) + invariant(array_abs_bound(r, j, start + len, bound)) + invariant(array_abs_bound(r, start + len, j + len, bound + MLKEM_Q)) + invariant(array_abs_bound(r, j + len, MLKEM_N, bound)) + decreases(start + len - j)) + { + int16_t t; + t = mlk_fqmul(r[j + len], zeta); + /* The precondition implies that the arithmetic does not overflow. */ + r[j + len] = (int16_t)(r[j] - t); + r[j] = (int16_t)(r[j] + t); + } +} + +/* + * Compute one layer of forward NTT + * Parameters: + * - r: Pointer to base of polynomial + * - layer: Variable indicating which layer is being applied. + */ + +/* Reference: Embedded in `ntt()` in the reference implementation @[REF]. */ +static void mlk_ntt_layer(int16_t r[MLKEM_N], unsigned layer) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(1 <= layer && layer <= 7) + requires(array_abs_bound(r, 0, MLKEM_N, layer * MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, (layer + 1) * MLKEM_Q))) +{ + unsigned start, k, len; + /* Twiddle factors for layer n are at indices 2^(n-1)..2^n-1. */ + k = 1u << (layer - 1); + len = (unsigned)MLKEM_N >> layer; + for (start = 0; start < MLKEM_N; start += 2 * len) + __loop__( + invariant(start < MLKEM_N + 2 * len) + invariant(k <= MLKEM_N / 2 && 2 * len * k == start + MLKEM_N) + invariant(array_abs_bound(r, 0, start, layer * MLKEM_Q + MLKEM_Q)) + invariant(array_abs_bound(r, start, MLKEM_N, layer * MLKEM_Q)) + decreases(MLKEM_N - start)) + { + int16_t zeta = mlk_zetas[k++]; + mlk_ntt_butterfly_block(r, zeta, start, len, layer * MLKEM_Q); + } +} + +/* + * Compute full forward NTT + * NOTE: This particular implementation satisfies a much tighter + * bound on the output coefficients (5*q) than the contractual one (8*q), + * but this is not needed in the calling code. Should we change the + * base multiplication strategy to require smaller NTT output bounds, + * the proof may need strengthening. + */ + +/* Reference: `ntt()` in the reference implementation @[REF]. + * - Iterate over `layer` instead of `len` in the outer loop + * to simplify computation of zeta index. */ +MLK_STATIC_TESTABLE void mlk_poly_ntt_c(mlk_poly *p) +__contract__( + requires(memory_no_alias(p, sizeof(mlk_poly))) + requires(array_abs_bound(p->coeffs, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(p, sizeof(mlk_poly))) + ensures(array_abs_bound(p->coeffs, 0, MLKEM_N, MLK_NTT_BOUND)) +) +{ + unsigned layer; + int16_t *r; + + mlk_assert_abs_bound(p, MLKEM_N, MLKEM_Q); + + r = p->coeffs; + + for (layer = 1; layer <= 7; layer++) + __loop__( + invariant(1 <= layer && layer <= 8) + invariant(array_abs_bound(r, 0, MLKEM_N, layer * MLKEM_Q)) + decreases(8 - layer)) + { + mlk_ntt_layer(r, layer); + } + + /* Check the stronger bound */ + mlk_assert_abs_bound(p, MLKEM_N, MLK_NTT_BOUND); +} + +MLK_INTERNAL_API +void mlk_poly_ntt(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_NTT) + int ret; + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_Q); + ret = mlk_ntt_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLK_NTT_BOUND); + return; + } +#endif /* MLK_USE_NATIVE_NTT */ + + mlk_poly_ntt_c(r); +} + + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Compute one layer of inverse NTT */ + +/* Reference: Embedded into `invntt()` in the reference implementation @[REF] */ +static void mlk_invntt_layer(int16_t *r, unsigned layer) +__contract__( + requires(memory_no_alias(r, sizeof(int16_t) * MLKEM_N)) + requires(1 <= layer && layer <= 7) + requires(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * MLKEM_N)) + ensures(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q))) +{ + unsigned start, k, len; + len = (unsigned)MLKEM_N >> layer; + k = (1u << layer) - 1; + for (start = 0; start < MLKEM_N; start += 2 * len) + __loop__( + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + invariant(start <= MLKEM_N && k <= 127) + /* Normalised form of k == MLKEM_N / len - 1 - start / (2 * len) */ + invariant(2 * len * k + start == 2 * MLKEM_N - 2 * len) + decreases(MLKEM_N - start)) + { + unsigned j; + int16_t zeta = mlk_zetas[k--]; + for (j = start; j < start + len; j++) + __loop__( + invariant(start <= j && j <= start + len) + invariant(start <= MLKEM_N && k <= 127) + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + decreases(start + len - j)) + { + int16_t t = r[j]; + /* The preconditions imply that the arithmetic does not overflow. */ + r[j] = mlk_barrett_reduce((int16_t)(t + r[j + len])); + r[j + len] = (int16_t)(r[j + len] - t); + r[j + len] = mlk_fqmul(r[j + len], zeta); + } + } +} + +/* Reference: `invntt()` in the reference implementation @[REF] + * - We normalize at the beginning of the inverse NTT, + * while the reference implementation normalizes at + * the end. This allows us to drop a call to `poly_reduce()` + * from the base multiplication. */ +MLK_STATIC_TESTABLE void mlk_poly_invntt_tomont_c(mlk_poly *p) +__contract__( + requires(memory_no_alias(p, sizeof(mlk_poly))) + assigns(memory_slice(p, sizeof(mlk_poly))) + ensures(array_abs_bound(p->coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND)) +) +{ + unsigned j, layer; + const int16_t f = 1441; /* check-magic: 1441 == pow(2,32 - 7,MLKEM_Q) */ + int16_t *r = p->coeffs; + + /* + * Scale input polynomial to account for Montgomery factor + * and NTT twist. This also brings coefficients down to + * absolute value < MLKEM_Q. + */ + for (j = 0; j < MLKEM_N; j++) + __loop__( + invariant(j <= MLKEM_N) + invariant(array_abs_bound(r, 0, j, MLKEM_Q)) + decreases(MLKEM_N - j)) + { + r[j] = mlk_fqmul(r[j], f); + } + + /* Run the invNTT layers */ + for (layer = 7; layer > 0; layer--) + __loop__( + invariant(0 <= layer && layer < 8) + invariant(array_abs_bound(r, 0, MLKEM_N, MLKEM_Q)) + decreases(layer)) + { + mlk_invntt_layer(r, layer); + } + + mlk_assert_abs_bound(p, MLKEM_N, MLK_INVNTT_BOUND); +} + +MLK_INTERNAL_API +void mlk_poly_invntt_tomont(mlk_poly *r) +{ +#if defined(MLK_USE_NATIVE_INTT) + int ret; + ret = mlk_intt_native(r->coeffs); + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + mlk_assert_abs_bound(r, MLKEM_N, MLK_INVNTT_BOUND); + return; + } +#endif /* MLK_USE_NATIVE_INTT */ + + mlk_poly_invntt_tomont_c(r); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(mlk_poly) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ diff --git a/src/mlkem-native/mlkem/src/poly.h b/src/mlkem-native/mlkem/src/poly.h new file mode 100644 index 0000000000..3def4394f1 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly.h @@ -0,0 +1,296 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_POLY_H +#define MLK_POLY_H + + +#include "cbmc.h" +#include "common.h" +#include "debug.h" +#include "verify.h" + +/* Absolute exclusive upper bound for the output of the inverse NTT */ +#define MLK_INVNTT_BOUND (8 * MLKEM_Q) + +/* Absolute exclusive upper bound for the output of the forward NTT */ +#define MLK_NTT_BOUND (8 * MLKEM_Q) + +/** + * Element of R_q = Z_q[X]/(X^n + 1). Represents polynomial + * coeffs[0] + X*coeffs[1] + X^2*coeffs[2] + ... + X^{n-1}*coeffs[n-1]. + */ +typedef struct +{ + int16_t coeffs[MLKEM_N]; /**< Polynomial coefficients. */ +} MLK_ALIGN mlk_poly; + +/** + * INTERNAL representation of precomputed data speeding up + * the base multiplication of two polynomials in NTT domain. + */ +typedef struct +{ + int16_t coeffs[MLKEM_N >> 1]; /**< Cached coefficients. */ +} MLK_ALIGN mlk_poly_mulcache; + +/** + * Generic Montgomery reduction; given a 32-bit integer a, computes a 16-bit + * integer congruent to a * R^-1 mod MLKEM_Q, where R=2^16. + * + * @param a Input integer to be reduced, of absolute value smaller or equal + * to INT32_MAX - 2^15 * MLKEM_Q. + * + * @return Integer congruent to a * R^-1 modulo MLKEM_Q, with absolute value + * <= ceil(|a| / 2^16) + (MLKEM_Q + 1)/2. + */ +static MLK_ALWAYS_INLINE int16_t mlk_montgomery_reduce(int32_t a) +__contract__( + requires(a < +(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q)) && + a > -(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q))) + /* We don't attempt to express an input-dependent output bound + * as the post-condition here. There are two call-sites for this + * function: + * - The base multiplication: Here, we need no output bound. + * - mlk_fqmul: Here, we inline this function and prove another spec + * for mlk_fqmul which does have a post-condition bound. */ +) +{ + /* check-magic: 62209 == unsigned_mod(pow(MLKEM_Q, -1, 2^16), 2^16) */ + const uint32_t QINV = 62209; + + /* Compute a*q^{-1} mod 2^16 in unsigned representatives. */ + const uint16_t a_reduced = mlk_cast_int32_to_uint16(a); + const uint16_t a_inverted = (a_reduced * QINV) & UINT16_MAX; + + /* Lift to signed canonical representative mod 2^16. */ + const int16_t t = mlk_cast_uint16_to_int16(a_inverted); + + int32_t r; + + mlk_assert(a < +(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q)) && + a > -(INT32_MAX - (((int32_t)1 << 15) * MLKEM_Q))); + + r = a - ((int32_t)t * MLKEM_Q); + + /* + * PORTABILITY: Right-shift on a signed integer is, strictly-speaking, + * implementation-defined for negative left argument. Here, + * we assume it's sign-preserving "arithmetic" shift right. (C99 6.5.7 (5)) + */ + r = r >> 16; + /* Bounds: |r >> 16| <= ceil(|r| / 2^16) + * <= ceil(|a| / 2^16 + MLKEM_Q / 2) + * <= ceil(|a| / 2^16) + (MLKEM_Q + 1) / 2 + * + * (Note that |a >> n| = ceil(|a| / 2^16) for negative a) + */ + return (int16_t)r; +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_poly_tomont MLK_NAMESPACE(poly_tomont) +/** + * In-place conversion of all coefficients of a polynomial from the normal + * domain to the Montgomery domain. + * + * Bounds: output < MLKEM_Q in absolute value. + * + * @spec{Internal normalization required in `mlk_indcpa_keypair_derand` as + * part of matrix-vector multiplication @[FIPS203, Algorithm 13, K-PKE.KeyGen, + * L18].} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_tomont(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#define mlk_poly_mulcache_compute MLK_NAMESPACE(poly_mulcache_compute) +/** + * Compute the mulcache for a polynomial in NTT domain. + * + * The mulcache of a degree-2 polynomial b := b0 + b1*X in Fq[X]/(X^2-zeta) + * is the value b1*zeta, needed when computing products of b in + * Fq[X]/(X^2-zeta). + * + * The mulcache of a polynomial in NTT domain -- which is a 128-tuple of + * degree-2 polynomials in Fq[X]/(X^2-zeta), for varying zeta, is the + * 128-tuple of mulcaches of those polynomials. + * + * @spec{Caches `b_1 * \gamma` in @[FIPS203, Algorithm 12, BaseCaseMultiply, + * L1].} + * + * @param[out] x Mulcache to be populated. + * @param[in] a Input polynomial. + */ +/* + * NOTE: The default C implementation of this function populates + * the mulcache with values in (-q,q), but this is not needed for the + * higher level safety proofs, and thus not part of the spec. + */ +MLK_INTERNAL_API +void mlk_poly_mulcache_compute(mlk_poly_mulcache *x, const mlk_poly *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_poly_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_poly))) + assigns(memory_slice(x, sizeof(mlk_poly_mulcache))) +); + +#define mlk_poly_reduce MLK_NAMESPACE(poly_reduce) +/** + * Convert a polynomial to unsigned canonical representatives. + * + * The input coefficients can be arbitrary integers in int16_t. The output + * coefficients are in [0,1,..,MLKEM_Q-1]. + * + * @spec{Normalizes on unsigned canonical representatives ahead of calling + * @[FIPS203, Compress_d, Eq (4.7)]. This is not made explicit in FIPS 203.} + * + * @param[in,out] r Input/output polynomial. + */ +/* + * NOTE: The semantics of mlk_poly_reduce() is different in + * the reference implementation, which requires + * signed canonical output data. Unsigned canonical + * outputs are better suited to the only remaining + * use of mlk_poly_reduce() in the context of (de)serialization. + */ +MLK_INTERNAL_API +void mlk_poly_reduce(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) +); + +#define mlk_poly_add MLK_NAMESPACE(poly_add) +/** + * Add two polynomials in place. + * + * The coefficients of @p r and @p b must be such that the addition does not + * overflow. Otherwise, the behaviour of this function is undefined. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L21].} + * + * @param[in,out] r Input-output polynomial to be added to. + * @param[in] b Input polynomial that should be added to @p r. Must be + * disjoint from @p r. + */ +/* + * NOTE: The reference implementation uses a 3-argument mlk_poly_add. + * We specialize to the accumulator form to avoid reasoning about aliasing. + */ +MLK_INTERNAL_API +void mlk_poly_add(mlk_poly *r, const mlk_poly *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(b, sizeof(mlk_poly))) + requires(forall(k0, 0, MLKEM_N, (int32_t) r->coeffs[k0] + b->coeffs[k0] <= INT16_MAX)) + requires(forall(k1, 0, MLKEM_N, (int32_t) r->coeffs[k1] + b->coeffs[k1] >= INT16_MIN)) + ensures(forall(k, 0, MLKEM_N, r->coeffs[k] == old(*r).coeffs[k] + b->coeffs[k])) + assigns(memory_slice(r, sizeof(mlk_poly))) +); + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_sub MLK_NAMESPACE(poly_sub) +/** + * Subtract two polynomials; no modular reduction is performed. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 15, K-PKE.Decrypt, L6].} + * + * @param[in,out] r Input-output polynomial to be subtracted from. + * @param[in] b Second input polynomial. + */ +/* + * NOTE: The reference implementation uses a 3-argument mlk_poly_sub. + * We specialize to the accumulator form to avoid reasoning about aliasing. + */ +MLK_INTERNAL_API +void mlk_poly_sub(mlk_poly *r, const mlk_poly *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(b, sizeof(mlk_poly))) + requires(forall(k0, 0, MLKEM_N, (int32_t) r->coeffs[k0] - b->coeffs[k0] <= INT16_MAX)) + requires(forall(k1, 0, MLKEM_N, (int32_t) r->coeffs[k1] - b->coeffs[k1] >= INT16_MIN)) + ensures(forall(k, 0, MLKEM_N, r->coeffs[k] == old(*r).coeffs[k] - b->coeffs[k])) + assigns(memory_slice(r, sizeof(mlk_poly))) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#define mlk_poly_ntt MLK_NAMESPACE(poly_ntt) +/** + * Compute the negacyclic number-theoretic transform (NTT) of a polynomial + * in place. + * + * The input is assumed to be in normal order and coefficient-wise bound by + * MLKEM_Q in absolute value. + * + * The output polynomial is in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set, and coefficient-wise bound + * by MLK_NTT_BOUND in absolute value. + * + * (NOTE: Sometimes the input to the NTT is actually smaller, which gives + * better bounds.) + * + * @spec{Implements @[FIPS203, Algorithm 9, NTT].} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_ntt(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_Q)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLK_NTT_BOUND)) +); + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_invntt_tomont MLK_NAMESPACE(poly_invntt_tomont) +/** + * Compute the inverse negacyclic number-theoretic transform (NTT) of a + * polynomial in place; input assumed to be in bitreversed order, output in + * normal order. + * + * The input is assumed to be in bitreversed order, or of a custom order if + * MLK_USE_NATIVE_NTT_CUSTOM_ORDER is set, and can have arbitrary + * coefficients in int16_t. + * + * The output polynomial is in normal order, and coefficient-wise bound by + * MLK_INVNTT_BOUND in absolute value. + * + * @spec{Implements composition of @[FIPS203, Algorithm 10, NTT^{-1}] and + * elementwise modular multiplication with a suitable Montgomery factor + * introduced during the base multiplication.} + * + * @param[in,out] r Input/output polynomial. + */ +MLK_INTERNAL_API +void mlk_poly_invntt_tomont(mlk_poly *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#endif /* !MLK_POLY_H */ diff --git a/src/mlkem-native/mlkem/src/poly_k.c b/src/mlkem-native/mlkem/src/poly_k.c new file mode 100644 index 0000000000..60380e34f3 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly_k.c @@ -0,0 +1,522 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [NeonNTT] + * Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1 + * Becker, Hwang, Kannwischer, Yang, Yang + * https://eprint.iacr.org/2021/986 + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "poly_k.h" + +#include "debug.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_poly_cbd_eta1 MLK_ADD_PARAM_SET(mlk_poly_cbd_eta1) +#define mlk_poly_cbd_eta2 MLK_ADD_PARAM_SET(mlk_poly_cbd_eta2) +#define mlk_polyvec_basemul_acc_montgomery_cached_c \ + MLK_ADD_PARAM_SET(mlk_polyvec_basemul_acc_montgomery_cached_c) +/* End of parameter set namespacing */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `polyvec_compress()` in the reference implementation @[REF] + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_INTERNAL_API +void mlk_polyvec_compress_du(uint8_t r[MLKEM_POLYVECCOMPRESSEDBYTES_DU], + const mlk_polyvec *a) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_compress_du(r + i * MLKEM_POLYCOMPRESSEDBYTES_DU, &a->vec[i]); + } +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `polyvec_decompress()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_decompress_du(mlk_polyvec *r, + const uint8_t a[MLKEM_POLYVECCOMPRESSEDBYTES_DU]) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_decompress_du(&r->vec[i], a + i * MLKEM_POLYCOMPRESSEDBYTES_DU); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +/* Reference: `polyvec_tobytes()` in the reference implementation @[REF]. + * - In contrast to the reference implementation, we assume + * unsigned canonical coefficients here. + * The reference implementation works with coefficients + * in the range [-(MLKEM_Q-1), MLKEM_Q-1]. */ +MLK_INTERNAL_API +void mlk_polyvec_tobytes(uint8_t r[MLKEM_POLYVECBYTES], const mlk_polyvec *a) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); + + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(r, MLKEM_POLYVECBYTES)) + invariant(i <= MLKEM_K) + decreases(MLKEM_K - i) + ) + { + mlk_poly_tobytes(&r[i * MLKEM_POLYBYTES], &a->vec[i]); + } +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/* Reference: `polyvec_frombytes()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_frombytes(mlk_polyvec *r, const uint8_t a[MLKEM_POLYVECBYTES]) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_frombytes(&r->vec[i], a + i * MLKEM_POLYBYTES); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* Reference: `polyvec_ntt()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_ntt(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_ntt(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLK_NTT_BOUND); +} + +/* Reference: `polyvec_invntt_tomont()` in the reference implementation @[REF]. + * - We normalize at the beginning of the inverse NTT, + * while the reference implementation normalizes at + * the end. This allows us to drop a call to `poly_reduce()` + * from the base multiplication. */ +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +MLK_INTERNAL_API +void mlk_polyvec_invntt_tomont(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_invntt_tomont(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLK_INVNTT_BOUND); +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +/* Reference: `polyvec_basemul_acc_montgomery()` in the + * reference implementation @[REF]. + * - We use a multiplication cache ('mulcache') here + * which is not present in the reference implementation @[REF]. + * This idea originates from @[NeonNTT] and is used + * at the C level here. + * - We compute the coefficients of the scalar product in 32-bit + * coefficients and perform only a single modular reduction + * at the end. The reference implementation uses 2 * MLKEM_K + * more modular reductions since it reduces after every modular + * multiplication. */ +MLK_STATIC_TESTABLE void mlk_polyvec_basemul_acc_montgomery_cached_c( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(memory_no_alias(b_cache, sizeof(mlk_polyvec_mulcache))) + requires(forall(k1, 0, MLKEM_K, + array_bound(a->vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) + assigns(memory_slice(r, sizeof(mlk_poly))) +) +{ + unsigned i; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); + + for (i = 0; i < MLKEM_N / 2; i++) + __loop__(invariant(i <= MLKEM_N / 2) + decreases(MLKEM_N / 2 - i)) + { + unsigned k; + int32_t t[2] = {0}; + for (k = 0; k < MLKEM_K; k++) + __loop__( + invariant(k <= MLKEM_K && + t[0] <= (int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768 && + t[0] >= - ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768) && + t[1] <= ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768) && + t[1] >= - ((int32_t) k * 2 * MLKEM_UINT12_LIMIT * 32768)) + decreases(MLKEM_K - k)) + { + t[0] += (int32_t)a->vec[k].coeffs[2 * i + 1] * b_cache->vec[k].coeffs[i]; + t[0] += (int32_t)a->vec[k].coeffs[2 * i] * b->vec[k].coeffs[2 * i]; + t[1] += (int32_t)a->vec[k].coeffs[2 * i] * b->vec[k].coeffs[2 * i + 1]; + t[1] += (int32_t)a->vec[k].coeffs[2 * i + 1] * b->vec[k].coeffs[2 * i]; + } + r->coeffs[2 * i + 0] = mlk_montgomery_reduce(t[0]); + r->coeffs[2 * i + 1] = mlk_montgomery_reduce(t[1]); + } +} + +MLK_INTERNAL_API +void mlk_polyvec_basemul_acc_montgomery_cached( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +{ +#if defined(MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED) + { + int ret; + mlk_assert_bound_2d(a->vec, MLKEM_K, MLKEM_N, 0, MLKEM_UINT12_LIMIT); +#if MLKEM_K == 2 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k2_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#elif MLKEM_K == 3 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k3_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#elif MLKEM_K == 4 + ret = mlk_polyvec_basemul_acc_montgomery_cached_k4_native( + r->coeffs, (const int16_t *)a, (const int16_t *)b, + (const int16_t *)b_cache); +#endif + if (ret == MLK_NATIVE_FUNC_SUCCESS) + { + return; + } + } +#endif /* MLK_USE_NATIVE_POLYVEC_BASEMUL_ACC_MONTGOMERY_CACHED */ + + mlk_polyvec_basemul_acc_montgomery_cached_c(r, a, b, b_cache); +} + +/* Reference: Does not exist in the reference implementation @[REF]. + * - The reference implementation does not use a + * multiplication cache ('mulcache'). This idea originates + * from @[NeonNTT] and is used at the C level here. */ +MLK_INTERNAL_API +void mlk_polyvec_mulcache_compute(mlk_polyvec_mulcache *x, const mlk_polyvec *a) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_mulcache_compute(&x->vec[i], &a->vec[i]); + } +} + +/* Reference: `polyvec_reduce()` in the reference implementation @[REF]. + * - We use _unsigned_ canonical outputs, while the reference + * implementation uses _signed_ canonical outputs. + * Accordingly, we need a conditional addition of MLKEM_Q + * here to go from signed to unsigned representatives. + * This conditional addition is then dropped from all + * polynomial compression functions instead (see `compress.c`). */ +MLK_INTERNAL_API +void mlk_polyvec_reduce(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_reduce(&r->vec[i]); + } + + mlk_assert_bound_2d(r->vec, MLKEM_K, MLKEM_N, 0, MLKEM_Q); +} + +/* Reference: `polyvec_add()` in the reference implementation @[REF]. + * - We use destructive version (output=first input) to avoid + * reasoning about aliasing in the CBMC specification */ +MLK_INTERNAL_API +void mlk_polyvec_add(mlk_polyvec *r, const mlk_polyvec *b) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + __loop__( + assigns(i, memory_slice(r, sizeof(mlk_polyvec))) + invariant(i <= MLKEM_K) + invariant(forall(j0, i, MLKEM_K, + forall(k0, 0, MLKEM_N, + ((int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] <= INT16_MAX) && + ((int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] >= INT16_MIN)))) + invariant(forall(j2, 0, i, + forall(k2, 0, MLKEM_N, + (r->vec[j2].coeffs[k2] <= INT16_MAX) && + (r->vec[j2].coeffs[k2] >= INT16_MIN)))) + decreases(MLKEM_K - i) + ) + { + mlk_poly_add(&r->vec[i], &b->vec[i]); + } +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +/* Reference: `polyvec_tomont()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_polyvec_tomont(mlk_polyvec *r) +{ + unsigned i; + for (i = 0; i < MLKEM_K; i++) + { + mlk_poly_tomont(&r->vec[i]); + } + + mlk_assert_abs_bound_2d(r->vec, MLKEM_K, MLKEM_N, MLKEM_Q); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + + +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter MLKEM_ETA1. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_eta1], where eta1 + * is specified per parameter set in @[FIPS203, Table 2] and represented as + * MLKEM_ETA1 here.} + * + * @reference{`poly_cbd_eta1` in the reference implementation @[REF].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +static MLK_INLINE void mlk_poly_cbd_eta1( + mlk_poly *r, const uint8_t buf[MLKEM_ETA1 * MLKEM_N / 4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(buf, MLKEM_ETA1 * MLKEM_N / 4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA1 + 1)) +) +{ +#if MLKEM_ETA1 == 2 + mlk_poly_cbd2(r, buf); +#elif MLKEM_ETA1 == 3 + mlk_poly_cbd3(r, buf); +#else +#error "Invalid value of MLKEM_ETA1" +#endif +} + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + (MLKEM_ETA1 == MLKEM_ETA2 && (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API))) +/* Reference: Does not exist in the reference implementation @[REF]. + * - This implements a x4-batched version of `poly_getnoise_eta1()` + * from the reference implementation, to leverage + * batched Keccak-f1600.*/ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, uint8_t nonce2, + uint8_t nonce3) +{ + MLK_ALIGN uint8_t buf[4][MLK_ALIGN_UP(MLKEM_ETA1 * MLKEM_N / 4)]; + MLK_ALIGN uint8_t extkey[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 1)]; + mlk_memcpy(extkey[0], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[1], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[2], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[3], seed, MLKEM_SYMBYTES); + extkey[0][MLKEM_SYMBYTES] = nonce0; + extkey[1][MLKEM_SYMBYTES] = nonce1; + extkey[2][MLKEM_SYMBYTES] = nonce2; + extkey[3][MLKEM_SYMBYTES] = nonce3; + +#if !defined(FIPS202_X4_DEFAULT_IMPLEMENTATION) && \ + !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + mlk_prf_eta1_x4(buf, extkey); +#else + mlk_prf_eta1(buf[0], extkey[0]); + mlk_prf_eta1(buf[1], extkey[1]); + mlk_prf_eta1(buf[2], extkey[2]); + if (r3 != NULL) + { + mlk_prf_eta1(buf[3], extkey[3]); + } +#endif /* !(!FIPS202_X4_DEFAULT_IMPLEMENTATION && \ + !MLK_CONFIG_SERIAL_FIPS202_ONLY) */ + + mlk_poly_cbd_eta1(r0, buf[0]); + mlk_poly_cbd_eta1(r1, buf[1]); + mlk_poly_cbd_eta1(r2, buf[2]); + if (r3 != NULL) + { + mlk_poly_cbd_eta1(r3, buf[3]); + mlk_assert_abs_bound(r3, MLKEM_N, MLKEM_ETA1 + 1); + } + + mlk_assert_abs_bound(r0, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r1, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r2, MLKEM_N, MLKEM_ETA1 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || (MLKEM_ETA1 == MLKEM_ETA2 && \ + (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API)) */ + +#if (MLKEM_K == 2 || MLKEM_K == 4) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter MLKEM_ETA2. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_eta2], where eta2 + * is specified per parameter set in @[FIPS203, Table 2] and represented as + * MLKEM_ETA2 here.} + * + * @reference{`poly_cbd_eta2` in the reference implementation @[REF].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +static MLK_INLINE void mlk_poly_cbd_eta2( + mlk_poly *r, const uint8_t buf[MLKEM_ETA2 * MLKEM_N / 4]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(buf, MLKEM_ETA2 * MLKEM_N / 4)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1))) +{ +#if MLKEM_ETA2 == 2 + mlk_poly_cbd2(r, buf); +#else +#error "Invalid value of MLKEM_ETA2" +#endif +} + +/* Reference: `poly_getnoise_eta2()` in the reference implementation @[REF]. + * - We include buffer zeroization. */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta2(mlk_poly *r, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce) +{ + MLK_ALIGN uint8_t buf[MLKEM_ETA2 * MLKEM_N / 4]; + MLK_ALIGN uint8_t extkey[MLKEM_SYMBYTES + 1]; + + mlk_memcpy(extkey, seed, MLKEM_SYMBYTES); + extkey[MLKEM_SYMBYTES] = nonce; + mlk_prf_eta2(buf, extkey); + + mlk_poly_cbd_eta2(r, buf); + + mlk_assert_abs_bound(r, MLKEM_N, MLKEM_ETA2 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* (MLKEM_K == 2 || MLKEM_K == 4) && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if MLKEM_K == 2 && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +/* Reference: Does not exist in the reference implementation @[REF]. + * - This implements a x4-batched version of `poly_getnoise_eta1()` + * and `poly_getnoise_eta2()` from the reference implementation, + * leveraging batched Keccak-f1600. + * - If a x4-batched Keccak-f1600 is available, we squeeze + * more random data than needed for the eta2 calls, to be + * be able to use a x4-batched Keccak-f1600. */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1122_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, + const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, + uint8_t nonce2, uint8_t nonce3) +{ +#if MLKEM_ETA2 >= MLKEM_ETA1 +#error mlk_poly_getnoise_eta1122_4x assumes MLKEM_ETA1 > MLKEM_ETA2 +#endif + MLK_ALIGN uint8_t buf[4][MLK_ALIGN_UP(MLKEM_ETA1 * MLKEM_N / 4)]; + MLK_ALIGN uint8_t extkey[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 1)]; + + mlk_memcpy(extkey[0], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[1], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[2], seed, MLKEM_SYMBYTES); + mlk_memcpy(extkey[3], seed, MLKEM_SYMBYTES); + extkey[0][MLKEM_SYMBYTES] = nonce0; + extkey[1][MLKEM_SYMBYTES] = nonce1; + extkey[2][MLKEM_SYMBYTES] = nonce2; + extkey[3][MLKEM_SYMBYTES] = nonce3; + + /* On systems with fast batched Keccak, we use 4-fold batched PRF, + * even though that means generating more random data in buf[2] and buf[3] + * than necessary. */ +#if !defined(FIPS202_X4_DEFAULT_IMPLEMENTATION) && \ + !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) + mlk_prf_eta1_x4(buf, extkey); +#else + mlk_prf_eta1(buf[0], extkey[0]); + mlk_prf_eta1(buf[1], extkey[1]); + mlk_prf_eta2(buf[2], extkey[2]); + mlk_prf_eta2(buf[3], extkey[3]); +#endif /* !(!FIPS202_X4_DEFAULT_IMPLEMENTATION && \ + !MLK_CONFIG_SERIAL_FIPS202_ONLY) */ + + mlk_poly_cbd_eta1(r0, buf[0]); + mlk_poly_cbd_eta1(r1, buf[1]); + mlk_poly_cbd_eta2(r2, buf[2]); + mlk_poly_cbd_eta2(r3, buf[3]); + + mlk_assert_abs_bound(r0, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r1, MLKEM_N, MLKEM_ETA1 + 1); + mlk_assert_abs_bound(r2, MLKEM_N, MLKEM_ETA2 + 1); + mlk_assert_abs_bound(r3, MLKEM_N, MLKEM_ETA2 + 1); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); + mlk_zeroize(extkey, sizeof(extkey)); +} +#endif /* MLKEM_K == 2 && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef mlk_poly_cbd_eta1 +#undef mlk_poly_cbd_eta2 +#undef mlk_polyvec_basemul_acc_montgomery_cached_c diff --git a/src/mlkem-native/mlkem/src/poly_k.h b/src/mlkem-native/mlkem/src/poly_k.h new file mode 100644 index 0000000000..1eba9fea67 --- /dev/null +++ b/src/mlkem-native/mlkem/src/poly_k.h @@ -0,0 +1,622 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_POLY_K_H +#define MLK_POLY_K_H + +#include "common.h" +#include "compress.h" +#include "poly.h" + +/* Parameter set namespacing + * This is to facilitate building multiple instances + * of mlkem-native (e.g. with varying parameter sets) + * within a single compilation unit. */ +#define mlk_polyvec MLK_ADD_PARAM_SET(mlk_polyvec) +#define mlk_polymat MLK_ADD_PARAM_SET(mlk_polymat) +#define mlk_polyvec_mulcache MLK_ADD_PARAM_SET(mlk_polyvec_mulcache) +/* End of parameter set namespacing */ + +/** Vector of MLKEM_K polynomials. */ +typedef struct +{ + mlk_poly vec[MLKEM_K]; /**< Component polynomials. */ +} MLK_ALIGN mlk_polyvec; + +/** MLKEM_K x MLKEM_K matrix of polynomials. */ +typedef struct +{ + mlk_polyvec vec[MLKEM_K]; /**< Rows of the matrix. */ +} MLK_ALIGN mlk_polymat; + +/** Vector of MLKEM_K mlk_poly_mulcache entries. */ +typedef struct +{ + mlk_poly_mulcache vec[MLKEM_K]; /**< Per-component caches. */ +} MLK_ALIGN mlk_polyvec_mulcache; + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_compress_du MLK_NAMESPACE_K(poly_compress_du) +/** + * Compression (du bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_{d_u} (Compress_{d_u} (u))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L22], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DU + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +static MLK_INLINE void mlk_poly_compress_du( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DU], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_DU))) +{ +#if MLKEM_DU == 10 + mlk_poly_compress_d10(r, a); +#elif MLKEM_DU == 11 + mlk_poly_compress_d11(r, a); +#else +#error "Invalid value of MLKEM_DU" +#endif +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_du MLK_NAMESPACE_K(poly_decompress_du) +/** + * De-serialization and subsequent decompression (du bits) of a polynomial; + * approximate inverse of mlk_poly_compress_du. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_{d_u} (ByteDecode_{d_u} (u))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L3], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DU + * bytes). + */ +static MLK_INLINE void mlk_poly_decompress_du( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_DU]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +{ +#if MLKEM_DU == 10 + mlk_poly_decompress_d10(r, a); +#elif MLKEM_DU == 11 + mlk_poly_decompress_d11(r, a); +#else +#error "Invalid value of MLKEM_DU" +#endif +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_compress_dv MLK_NAMESPACE_K(poly_compress_dv) +/** + * Compression (dv bits) and subsequent serialization of a polynomial. + * + * @spec{Implements `ByteEncode_{d_v} (Compress_{d_v} (v))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L23], with level-specific d_v defined in + * @[FIPS203, Table 2], and given by MLKEM_DV here.} + * + * @param[out] r Output byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DV + * bytes). + * @param[in] a Input polynomial. Coefficients must be unsigned canonical, + * i.e. in [0,1,..,MLKEM_Q-1]. + */ +static MLK_INLINE void mlk_poly_compress_dv( + uint8_t r[MLKEM_POLYCOMPRESSEDBYTES_DV], const mlk_poly *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYCOMPRESSEDBYTES_DV)) + requires(memory_no_alias(a, sizeof(mlk_poly))) + requires(array_bound(a->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + assigns(memory_slice(r, MLKEM_POLYCOMPRESSEDBYTES_DV))) +{ +#if MLKEM_DV == 4 + mlk_poly_compress_d4(r, a); +#elif MLKEM_DV == 5 + mlk_poly_compress_d5(r, a); +#else +#error "Invalid value of MLKEM_DV" +#endif +} +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_decompress_dv MLK_NAMESPACE_K(poly_decompress_dv) +/** + * De-serialization and subsequent decompression (dv bits) of a polynomial; + * approximate inverse of mlk_poly_compress_dv. + * + * Upon return, the coefficients of the output polynomial are + * unsigned-canonical (non-negative and smaller than MLKEM_Q). + * + * @spec{Implements `Decompress_{d_v} (ByteDecode_{d_v} (v))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L4], with level-specific d_v defined in + * @[FIPS203, Table 2], and given by MLKEM_DV here.} + * + * @param[out] r Output polynomial. + * @param[in] a Input byte array (of length MLKEM_POLYCOMPRESSEDBYTES_DV + * bytes). + */ +static MLK_INLINE void mlk_poly_decompress_dv( + mlk_poly *r, const uint8_t a[MLKEM_POLYCOMPRESSEDBYTES_DV]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYCOMPRESSEDBYTES_DV)) + requires(memory_no_alias(r, sizeof(mlk_poly))) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_bound(r->coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +{ +#if MLKEM_DV == 4 + mlk_poly_decompress_d4(r, a); +#elif MLKEM_DV == 5 + mlk_poly_decompress_d5(r, a); +#else +#error "Invalid value of MLKEM_DV" +#endif +} +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_compress_du MLK_NAMESPACE_K(polyvec_compress_du) +/** + * Compress and serialize a vector of polynomials. + * + * @spec{Implements `ByteEncode_{d_u} (Compress_{d_u} (u))` in @[FIPS203, + * Algorithm 14 (K-PKE.Encrypt), L22], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output byte array (needs space for + * MLKEM_POLYVECCOMPRESSEDBYTES_DU bytes). + * @param[in] a Input vector of polynomials. Coefficients must be unsigned + * canonical, i.e. in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_polyvec_compress_du(uint8_t r[MLKEM_POLYVECCOMPRESSEDBYTES_DU], + const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(r, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(forall(k0, 0, MLKEM_K, + array_bound(a->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(r, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_decompress_du MLK_NAMESPACE_K(polyvec_decompress_du) +/** + * De-serialize and decompress a vector of polynomials; approximate inverse + * of mlk_polyvec_compress_du. + * + * @spec{Implements `Decompress_{d_u} (ByteDecode_{d_u} (u))` in @[FIPS203, + * Algorithm 15 (K-PKE.Decrypt), L3], with level-specific d_u defined in + * @[FIPS203, Table 2], and given by MLKEM_DU here.} + * + * @param[out] r Output vector of polynomials. Coefficients are normalized + * to [0,1,..,MLKEM_Q-1]. + * @param[in] a Input byte array (of length MLKEM_POLYVECCOMPRESSEDBYTES_DU + * bytes). + */ +MLK_INTERNAL_API +void mlk_polyvec_decompress_du(mlk_polyvec *r, + const uint8_t a[MLKEM_POLYVECCOMPRESSEDBYTES_DU]) +__contract__( + requires(memory_no_alias(a, MLKEM_POLYVECCOMPRESSEDBYTES_DU)) + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +); +#endif /* !MLK_CONFIG_NO_DECAPS_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || !defined(MLK_CONFIG_NO_ENCAPS_API) +#define mlk_polyvec_tobytes MLK_NAMESPACE_K(polyvec_tobytes) +/** + * Serialize a vector of polynomials. + * + * @spec{Implements ByteEncode_12 @[FIPS203, Algorithm 5]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays] and + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[out] r Output byte array (needs space for MLKEM_POLYVECBYTES bytes). + * @param[in] a Input vector of polynomials. Each polynomial must have + * coefficients in [0,1,..,MLKEM_Q-1]. + */ +MLK_INTERNAL_API +void mlk_polyvec_tobytes(uint8_t r[MLKEM_POLYVECBYTES], const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(r, MLKEM_POLYVECBYTES)) + requires(forall(k0, 0, MLKEM_K, + array_bound(a->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) + assigns(memory_slice(r, MLKEM_POLYVECBYTES)) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || !MLK_CONFIG_NO_ENCAPS_API */ + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_frombytes MLK_NAMESPACE_K(polyvec_frombytes) +/** + * De-serialize a vector of polynomials; inverse of mlk_polyvec_tobytes. + * + * @spec{Implements ByteDecode_12 @[FIPS203, Algorithm 6]. Extended to + * vectors as per @[FIPS203, 2.4.8 Applying Algorithms to Arrays] and + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[out] r Output vector of polynomials. Coefficients will be + * normalized in [0,1,..,4095]. + * @param[in] a Input byte array (of length MLKEM_POLYVECBYTES bytes). + */ +MLK_INTERNAL_API +void mlk_polyvec_frombytes(mlk_polyvec *r, const uint8_t a[MLKEM_POLYVECBYTES]) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(memory_no_alias(a, MLKEM_POLYVECBYTES)) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#define mlk_polyvec_ntt MLK_NAMESPACE_K(polyvec_ntt) +/** + * Apply forward NTT to all elements of a vector of polynomials. + * + * The input is assumed to be in normal order and coefficient-wise bound by + * MLKEM_Q in absolute value. + * + * The output polynomial is in bitreversed order, and coefficient-wise bound + * by MLK_NTT_BOUND in absolute value. + * + * @spec{Implements @[FIPS203, Algorithm 9, NTT]. Extended to vectors as per + * @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[in,out] r Input/output vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_ntt(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLKEM_Q))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLK_NTT_BOUND))) +); + +#if !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_polyvec_invntt_tomont MLK_NAMESPACE_K(polyvec_invntt_tomont) +/** + * Apply inverse NTT to all elements of a vector of polynomials and multiply + * by Montgomery factor 2^16. + * + * The input is assumed to be in bitreversed order, and can have arbitrary + * coefficients in int16_t. + * + * The output polynomial is in normal order, and coefficient-wise bound by + * MLK_INVNTT_BOUND in absolute value. + * + * @spec{Implements @[FIPS203, Algorithm 10, NTT^{-1}]. Extended to vectors + * as per @[FIPS203, 2.4.6, Matrices and Vectors].} + * + * @param[in,out] r Input/output vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_invntt_tomont(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLK_INVNTT_BOUND))) +); +#endif /* !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#define mlk_polyvec_basemul_acc_montgomery_cached \ + MLK_NAMESPACE_K(polyvec_basemul_acc_montgomery_cached) +/** + * Scalar product of two vectors of polynomials in NTT domain, using + * mulcache for the second operand. + * + * Bounds: every coefficient of @p a is assumed to be in [0,1,..,4095]. No + * bounds guarantees for the coefficients in the result. + * + * @spec{Implements @[FIPS203, Section 2.4.7, Eq (2.14)], @[FIPS203, + * Algorithm 11, MultiplyNTTs], and @[FIPS203, Algorithm 12, + * BaseCaseMultiply].} + * + * @param[out] r Output polynomial. + * @param[in] a First input polynomial vector. + * @param[in] b Second input polynomial vector. + * @param[in] b_cache Mulcache for the second input polynomial vector. Can + * be computed via mlk_polyvec_mulcache_compute(). + */ +MLK_INTERNAL_API +void mlk_polyvec_basemul_acc_montgomery_cached( + mlk_poly *r, const mlk_polyvec *a, const mlk_polyvec *b, + const mlk_polyvec_mulcache *b_cache) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(memory_no_alias(b_cache, sizeof(mlk_polyvec_mulcache))) + requires(forall(k1, 0, MLKEM_K, + array_bound(a->vec[k1].coeffs, 0, MLKEM_N, 0, MLKEM_UINT12_LIMIT))) + assigns(memory_slice(r, sizeof(mlk_poly))) +); + +#define mlk_polyvec_mulcache_compute MLK_NAMESPACE_K(polyvec_mulcache_compute) +/** + * Compute the mulcache for a vector of polynomials in NTT domain. + * + * The mulcache of a degree-2 polynomial b := b0 + b1*X in Fq[X]/(X^2-zeta) + * is the value b1*zeta, needed when computing products of b in + * Fq[X]/(X^2-zeta). + * + * The mulcache of a polynomial in NTT domain -- which is a 128-tuple of + * degree-2 polynomials in Fq[X]/(X^2-zeta), for varying zeta, is the + * 128-tuple of mulcaches of those polynomials. + * + * The mulcache of a vector of polynomials is the vector of mulcaches of + * its entries. + * + * @spec{Caches `b_1 * \gamma` in @[FIPS203, Algorithm 12, BaseCaseMultiply, + * L1].} + * + * @param[out] x Mulcache to be populated. + * @param[in] a Input polynomial vector. + */ +/* + * NOTE: The default C implementation of this function populates + * the mulcache with values in (-q,q), but this is not needed for the + * higher level safety proofs, and thus not part of the spec. + */ +MLK_INTERNAL_API +void mlk_polyvec_mulcache_compute(mlk_polyvec_mulcache *x, const mlk_polyvec *a) +__contract__( + requires(memory_no_alias(x, sizeof(mlk_polyvec_mulcache))) + requires(memory_no_alias(a, sizeof(mlk_polyvec))) + assigns(memory_slice(x, sizeof(mlk_polyvec_mulcache))) +); + +#define mlk_polyvec_reduce MLK_NAMESPACE_K(polyvec_reduce) +/** + * Apply Barrett reduction to each coefficient of each element of a vector + * of polynomials. For details of the Barrett reduction see comments in + * poly.c. + * + * @spec{Normalizes on unsigned canonical representatives ahead of calling + * @[FIPS203, Compress_d, Eq (4.7)]. This is not made explicit in FIPS 203.} + * + * @param[in,out] r Input/output polynomial vector. + */ +/* + * NOTE: The semantics of mlk_polyvec_reduce() is different in + * the reference implementation, which requires + * signed canonical output data. Unsigned canonical + * outputs are better suited to the only remaining + * use of mlk_poly_reduce() in the context of (de)serialization. + */ +MLK_INTERNAL_API +void mlk_polyvec_reduce(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(k0, 0, MLKEM_K, + array_bound(r->vec[k0].coeffs, 0, MLKEM_N, 0, MLKEM_Q))) +); + +#define mlk_polyvec_add MLK_NAMESPACE_K(polyvec_add) +/** + * Add vectors of polynomials. + * + * The coefficients of @p r and @p b must be such that the addition does + * not overflow. Otherwise, the behaviour of this function is undefined. + * + * The coefficients returned in @p *r are in int16_t which is sufficient to + * prove type-safety of calling units. Therefore, no stronger ensures clause + * is required on this function. + * + * @spec{@[FIPS203, 2.4.5, Arithmetic With Polynomials and NTT + * Representations]. Used in @[FIPS203, Algorithm 14 (K-PKE.Encrypt), L19].} + * + * @param[in,out] r Input-output vector of polynomials to be added to. + * @param[in] b Second input vector of polynomials. + */ +MLK_INTERNAL_API +void mlk_polyvec_add(mlk_polyvec *r, const mlk_polyvec *b) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + requires(memory_no_alias(b, sizeof(mlk_polyvec))) + requires(forall(j0, 0, MLKEM_K, + forall(k0, 0, MLKEM_N, + (int32_t)r->vec[j0].coeffs[k0] + b->vec[j0].coeffs[k0] <= INT16_MAX))) + requires(forall(j1, 0, MLKEM_K, + forall(k1, 0, MLKEM_N, + (int32_t)r->vec[j1].coeffs[k1] + b->vec[j1].coeffs[k1] >= INT16_MIN))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) +); + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) +#define mlk_polyvec_tomont MLK_NAMESPACE_K(polyvec_tomont) +/** + * In-place conversion of all coefficients of a polynomial vector from the + * normal domain to the Montgomery domain. + * + * Bounds: output < MLKEM_Q in absolute value. + * + * @spec{Internal normalization required in `mlk_indcpa_keypair_derand` as + * part of matrix-vector multiplication @[FIPS203, Algorithm 13, K-PKE.KeyGen, + * L18].} + * + * @param[in,out] r Input/output polynomial vector. + */ +MLK_INTERNAL_API +void mlk_polyvec_tomont(mlk_polyvec *r) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_polyvec))) + assigns(memory_slice(r, sizeof(mlk_polyvec))) + ensures(forall(j, 0, MLKEM_K, + array_abs_bound(r->vec[j].coeffs, 0, MLKEM_N, MLKEM_Q))) +); +#endif /* !MLK_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLK_CONFIG_NO_KEYPAIR_API) || \ + (MLKEM_ETA1 == MLKEM_ETA2 && (!defined(MLK_CONFIG_NO_ENCAPS_API) || \ + !defined(MLK_CONFIG_NO_DECAPS_API))) +#define mlk_poly_getnoise_eta1_4x MLK_NAMESPACE_K(poly_getnoise_eta1_4x) +/** + * Batch sample four polynomials deterministically from a seed and nonces, + * with output polynomials close to centered binomial distribution with + * parameter MLKEM_ETA1. + * + * @spec{Implements 4x `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))` appears in + * @[FIPS203, Algorithm 13, K-PKE.KeyGen, L{9, 13}] and @[FIPS203, + * Algorithm 14, K-PKE.Encrypt, L10].} + * + * @param[out] r0 Output polynomial. + * @param[out] r1 Output polynomial. + * @param[out] r2 Output polynomial. + * @param[out] r3 Output polynomial. May be NULL. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce0 One-byte input nonce. + * @param nonce1 One-byte input nonce. + * @param nonce2 One-byte input nonce. + * @param nonce3 One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, uint8_t nonce2, + uint8_t nonce3) +__contract__( + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + requires(memory_no_alias(r0, sizeof(mlk_poly))) + requires(memory_no_alias(r1, sizeof(mlk_poly))) + requires(memory_no_alias(r2, sizeof(mlk_poly))) + requires(r3 == NULL || memory_no_alias(r3, sizeof(mlk_poly))) + assigns(memory_slice(r0, sizeof(mlk_poly))) + assigns(memory_slice(r1, sizeof(mlk_poly))) + assigns(memory_slice(r2, sizeof(mlk_poly))) + assigns(r3 != NULL: memory_slice(r3, sizeof(mlk_poly))) + ensures(array_abs_bound(r0->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(array_abs_bound(r1->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(array_abs_bound(r2->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) + ensures(r3 != NULL ==> array_abs_bound(r3->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1)) +); + +#if MLKEM_ETA1 == MLKEM_ETA2 +/* + * We only require mlk_poly_getnoise_eta2_4x for ml-kem-768 and ml-kem-1024 + * where MLKEM_ETA2 = MLKEM_ETA1 = 2. + * For ml-kem-512, mlk_poly_getnoise_eta1122_4x is used instead. + */ +#define mlk_poly_getnoise_eta2_4x mlk_poly_getnoise_eta1_4x +#endif /* MLKEM_ETA1 == MLKEM_ETA2 */ +#endif /* !MLK_CONFIG_NO_KEYPAIR_API || (MLKEM_ETA1 == MLKEM_ETA2 && \ + (!MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API)) */ + +#if (MLKEM_K == 2 || MLKEM_K == 4) && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +#define mlk_poly_getnoise_eta2 MLK_NAMESPACE_K(poly_getnoise_eta2) +/** + * Sample a polynomial deterministically from a seed and a nonce, with + * output polynomial close to centered binomial distribution with parameter + * MLKEM_ETA2. + * + * @spec{Implements `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))` appears in + * @[FIPS203, Algorithm 14, K-PKE.Encrypt, L14].} + * + * @param[out] r Output polynomial. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta2(mlk_poly *r, const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce) +__contract__( + requires(memory_no_alias(r, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(r, sizeof(mlk_poly))) + ensures(array_abs_bound(r->coeffs, 0, MLKEM_N, MLKEM_ETA2 + 1)) +); +#endif /* (MLKEM_K == 2 || MLKEM_K == 4) && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#if MLKEM_K == 2 && \ + (!defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API)) +#define mlk_poly_getnoise_eta1122_4x MLK_NAMESPACE_K(poly_getnoise_eta1122_4x) +/** + * Batch sample four polynomials deterministically from a seed and nonces, + * with output polynomials close to centered binomial distribution with + * parameter MLKEM_ETA1 and MLKEM_ETA2. + * + * @spec{Implements two instances each of + * `SamplePolyCBD_{eta1} (PRF_{eta1} (sigma, N))` and + * `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))`: + * @[FIPS203, Algorithm 8, SamplePolyCBD_eta] and @[FIPS203, Eq (4.3), + * PRF_eta]. `SamplePolyCBD_{eta2} (PRF_{eta2} (sigma, N))` appears in + * @[FIPS203, Algorithm 14, K-PKE.Encrypt, L14].} + * + * @param[out] r0 Output polynomial. + * @param[out] r1 Output polynomial. + * @param[out] r2 Output polynomial. + * @param[out] r3 Output polynomial. + * @param[in] seed Input seed (of length MLKEM_SYMBYTES bytes). + * @param nonce0 One-byte input nonce. + * @param nonce1 One-byte input nonce. + * @param nonce2 One-byte input nonce. + * @param nonce3 One-byte input nonce. + */ +MLK_INTERNAL_API +void mlk_poly_getnoise_eta1122_4x(mlk_poly *r0, mlk_poly *r1, mlk_poly *r2, + mlk_poly *r3, + const uint8_t seed[MLKEM_SYMBYTES], + uint8_t nonce0, uint8_t nonce1, + uint8_t nonce2, uint8_t nonce3) +__contract__( + requires(memory_no_alias(r0, sizeof(mlk_poly))) + requires(memory_no_alias(r1, sizeof(mlk_poly))) + requires(memory_no_alias(r2, sizeof(mlk_poly))) + requires(memory_no_alias(r3, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES)) + assigns(memory_slice(r0, sizeof(mlk_poly))) + assigns(memory_slice(r1, sizeof(mlk_poly))) + assigns(memory_slice(r2, sizeof(mlk_poly))) + assigns(memory_slice(r3, sizeof(mlk_poly))) + ensures(array_abs_bound(r0->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1) + && array_abs_bound(r1->coeffs,0, MLKEM_N, MLKEM_ETA1 + 1) + && array_abs_bound(r2->coeffs,0, MLKEM_N, MLKEM_ETA2 + 1) + && array_abs_bound(r3->coeffs,0, MLKEM_N, MLKEM_ETA2 + 1)) +); +#endif /* MLKEM_K == 2 && (!MLK_CONFIG_NO_ENCAPS_API || \ + !MLK_CONFIG_NO_DECAPS_API) */ + +#endif /* !MLK_POLY_K_H */ diff --git a/src/mlkem-native/mlkem/src/randombytes.h b/src/mlkem-native/mlkem/src/randombytes.h new file mode 100644 index 0000000000..46274422b0 --- /dev/null +++ b/src/mlkem-native/mlkem/src/randombytes.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_RANDOMBYTES_H +#define MLK_RANDOMBYTES_H + + +#include "cbmc.h" +#include "common.h" + +#if !defined(MLK_CONFIG_NO_RANDOMIZED_API) +#if !defined(MLK_CONFIG_CUSTOM_RANDOMBYTES) +/** + * Fill a buffer with cryptographically secure random bytes. + * + * mlkem-native does not provide an implementation of this function. + * It must be provided by the consumer. + * + * To use a custom random byte source with a different name or signature, + * set MLK_CONFIG_CUSTOM_RANDOMBYTES and define mlk_randombytes directly. + * + * @param[out] out Output buffer. + * @param outlen Number of random bytes to write. + * + * @retval 0 Success. + * @retval other Failure; top-level APIs propagate this as MLK_ERR_RNG_FAIL. + */ +int randombytes(uint8_t *out, size_t outlen); + +/** + * Internal wrapper around randombytes(). + * + * Fills a buffer with cryptographically secure random bytes. + * + * This function can be replaced by setting MLK_CONFIG_CUSTOM_RANDOMBYTES + * and defining mlk_randombytes directly. + * + * @param[out] out Output buffer. + * @param outlen Number of random bytes to write. + * + * @retval 0 Success. + * @retval other Failure; top-level APIs propagate this as MLK_ERR_RNG_FAIL. + */ +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_randombytes(uint8_t *out, size_t outlen) +__contract__( + requires(memory_no_alias(out, outlen)) + assigns(memory_slice(out, outlen))) { return randombytes(out, outlen); } +#endif /* !MLK_CONFIG_CUSTOM_RANDOMBYTES */ +#endif /* !MLK_CONFIG_NO_RANDOMIZED_API */ +#endif /* !MLK_RANDOMBYTES_H */ diff --git a/src/mlkem-native/mlkem/src/sampling.c b/src/mlkem-native/mlkem/src/sampling.c new file mode 100644 index 0000000000..b01ac5619e --- /dev/null +++ b/src/mlkem-native/mlkem/src/sampling.c @@ -0,0 +1,371 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + */ + +#include "common.h" +#if !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) + +#include "debug.h" +#include "sampling.h" +#include "symmetric.h" +#include "verify.h" + +/* Reference: `rej_uniform()` in the reference implementation @[REF]. + * - Our signature differs from the reference implementation + * in that it adds the offset and always expects the base of the + * target buffer. This avoids shifting the buffer base in the + * caller, which appears tricky to reason about. */ +MLK_STATIC_TESTABLE unsigned mlk_rej_uniform_c(int16_t *r, unsigned target, + unsigned offset, + const uint8_t *buf, + unsigned buflen) +__contract__( + requires(offset <= target && target <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * target)) + requires(memory_no_alias(buf, buflen)) + requires(array_bound(r, 0, offset, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * target)) + ensures(offset <= return_value && return_value <= target) + ensures(array_bound(r, 0, return_value, 0, MLKEM_Q))) +{ + unsigned ctr, pos; + int16_t val0, val1; + + mlk_assert_bound(r, offset, 0, MLKEM_Q); + + ctr = offset; + pos = 0; + /* pos + 3 cannot overflow due to the assumption buflen <= 4096 */ + while (ctr < target && pos + 3 <= buflen) + __loop__( + invariant(offset <= ctr && ctr <= target && pos <= buflen) + invariant(array_bound(r, 0, ctr, 0, MLKEM_Q)) + decreases(buflen - pos)) + { + /* Safety: + * - The explicit cast to uint16_t ensures that << 8 does + * not signed-overflow even on a 16-bit system. + * - The conversion to int16_t is safe due to the explicit 0xFFF + * truncation. + */ + val0 = (int16_t)(((buf[pos + 0] >> 0) | ((uint16_t)buf[pos + 1] << 8)) & + 0xFFF); + val1 = (int16_t)(((buf[pos + 1] >> 4) | (buf[pos + 2] << 4)) & 0xFFF); + pos += 3; + + if (val0 < MLKEM_Q) + { + r[ctr++] = val0; + } + if (ctr < target && val1 < MLKEM_Q) + { + r[ctr++] = val1; + } + } + + mlk_assert_bound(r, ctr, 0, MLKEM_Q); + return ctr; +} + +/** + * Run rejection sampling on uniform random bytes to generate uniform random + * integers mod MLKEM_Q. + * + * @reference{`rej_uniform()` in the reference implementation @[REF]. Our + * signature differs from the reference in that it adds the offset and always + * expects the base of the target buffer; this avoids shifting the buffer + * base in the caller, which is tricky to reason about. Has an optional + * fallback to a native implementation.} + * + * @param[out] r Output buffer. + * @param target Requested number of 16-bit integers (uniform mod MLKEM_Q). + * Must be <= 4096. + * @param offset Number of 16-bit integers that have already been + * sampled. Must be <= @p target. + * @param[in] buf Input buffer (assumed to be uniform random bytes). + * @param buflen Length of input buffer in bytes. Must be <= 4096 and a + * multiple of 3. + * + * @note Strictly speaking, only a few values of @p buflen near UINT_MAX need + * excluding. The limit of 4096 is somewhat arbitrary but sufficient + * for all uses of this function. Similarly, the actual limit for + * @p target is UINT_MAX/2. + * + * @return New offset of sampled 16-bit integers, at most @p target and at + * least the initial @p offset. If the new offset is strictly less + * than @p target, the entire input buffer is guaranteed to have been + * consumed; otherwise no information is provided on how many bytes + * of the input buffer have been consumed. + */ +static unsigned mlk_rej_uniform(int16_t *r, unsigned target, unsigned offset, + const uint8_t *buf, unsigned buflen) +__contract__( + requires(offset <= target && target <= 4096 && buflen <= 4096 && buflen % 3 == 0) + requires(memory_no_alias(r, sizeof(int16_t) * target)) + requires(memory_no_alias(buf, buflen)) + requires(array_bound(r, 0, offset, 0, MLKEM_Q)) + assigns(memory_slice(r, sizeof(int16_t) * target)) + ensures(offset <= return_value && return_value <= target) + ensures(array_bound(r, 0, return_value, 0, MLKEM_Q)) +) +{ +#if defined(MLK_USE_NATIVE_REJ_UNIFORM) + if (offset == 0) + { + int ret; + ret = mlk_rej_uniform_native(r, target, buf, buflen); + if (ret != MLK_NATIVE_FUNC_FALLBACK) + { + unsigned res = (unsigned)ret; + mlk_assert_bound(r, res, 0, MLKEM_Q); + return res; + } + } +#endif /* MLK_USE_NATIVE_REJ_UNIFORM */ + + return mlk_rej_uniform_c(r, target, offset, buf, buflen); +} + +#ifndef MLKEM_GEN_MATRIX_NBLOCKS +#define MLKEM_GEN_MATRIX_NBLOCKS \ + ((12 * MLKEM_N / 8 * ((uint32_t)1 << 12) / MLKEM_Q + MLK_XOF_RATE) / \ + MLK_XOF_RATE) +#endif + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +/* Reference: Does not exist in the reference implementation @[REF]. + * - x4-batched version of `rej_uniform()` from the + * reference implementation, leveraging x4-batched Keccak-f1600. */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform_x4(mlk_poly *vec0, mlk_poly *vec1, mlk_poly *vec2, + mlk_poly *vec3, + uint8_t seed[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]) +{ + /* Temporary buffers for XOF output before rejection sampling */ + MLK_ALIGN uint8_t + buf[4][MLK_ALIGN_UP(MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE)]; + + /* Tracks the number of coefficients we have already sampled */ + unsigned ctr[4]; + mlk_xof_x4_ctx statex; + unsigned buflen; + + mlk_xof_x4_init(&statex); + mlk_xof_x4_absorb(&statex, seed, MLKEM_SYMBYTES + 2); + + /* + * Initially, squeeze heuristic number of MLKEM_GEN_MATRIX_NBLOCKS. + * This should generate the matrix entries with high probability. + */ + mlk_xof_x4_squeezeblocks(buf, MLKEM_GEN_MATRIX_NBLOCKS, &statex); + buflen = MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE; + ctr[0] = mlk_rej_uniform(vec0->coeffs, MLKEM_N, 0, buf[0], buflen); + ctr[1] = mlk_rej_uniform(vec1->coeffs, MLKEM_N, 0, buf[1], buflen); + ctr[2] = mlk_rej_uniform(vec2->coeffs, MLKEM_N, 0, buf[2], buflen); + ctr[3] = mlk_rej_uniform(vec3->coeffs, MLKEM_N, 0, buf[3], buflen); + + /* + * So long as not all matrix entries have been generated, squeeze + * one more block a time until we're done. + */ + buflen = MLK_XOF_RATE; + while (ctr[0] < MLKEM_N || ctr[1] < MLKEM_N || ctr[2] < MLKEM_N || + ctr[3] < MLKEM_N) + __loop__( + assigns(ctr, statex, + memory_slice(vec0, sizeof(mlk_poly)), + memory_slice(vec1, sizeof(mlk_poly)), + memory_slice(vec2, sizeof(mlk_poly)), + memory_slice(vec3, sizeof(mlk_poly)), + object_whole(buf)) + invariant(ctr[0] <= MLKEM_N && ctr[1] <= MLKEM_N) + invariant(ctr[2] <= MLKEM_N && ctr[3] <= MLKEM_N) + invariant(array_bound(vec0->coeffs, 0, ctr[0], 0, MLKEM_Q)) + invariant(array_bound(vec1->coeffs, 0, ctr[1], 0, MLKEM_Q)) + invariant(array_bound(vec2->coeffs, 0, ctr[2], 0, MLKEM_Q)) + invariant(array_bound(vec3->coeffs, 0, ctr[3], 0, MLKEM_Q))) + { + mlk_xof_x4_squeezeblocks(buf, 1, &statex); + ctr[0] = mlk_rej_uniform(vec0->coeffs, MLKEM_N, ctr[0], buf[0], buflen); + ctr[1] = mlk_rej_uniform(vec1->coeffs, MLKEM_N, ctr[1], buf[1], buflen); + ctr[2] = mlk_rej_uniform(vec2->coeffs, MLKEM_N, ctr[2], buf[2], buflen); + ctr[3] = mlk_rej_uniform(vec3->coeffs, MLKEM_N, ctr[3], buf[3], buflen); + } + + mlk_xof_x4_release(&statex); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); +} +#endif /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +MLK_INTERNAL_API +void mlk_poly_rej_uniform(mlk_poly *entry, uint8_t seed[MLKEM_SYMBYTES + 2]) +{ + mlk_xof_ctx state; + MLK_ALIGN uint8_t buf[MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE]; + unsigned ctr, buflen; + + mlk_xof_init(&state); + mlk_xof_absorb(&state, seed, MLKEM_SYMBYTES + 2); + + /* Initially, squeeze + sample heuristic number of MLKEM_GEN_MATRIX_NBLOCKS. + */ + /* This should generate the matrix entry with high probability. */ + mlk_xof_squeezeblocks(buf, MLKEM_GEN_MATRIX_NBLOCKS, &state); + buflen = MLKEM_GEN_MATRIX_NBLOCKS * MLK_XOF_RATE; + ctr = mlk_rej_uniform(entry->coeffs, MLKEM_N, 0, buf, buflen); + + /* Squeeze + sample one more block a time until we're done */ + buflen = MLK_XOF_RATE; + while (ctr < MLKEM_N) + __loop__( + assigns(ctr, state, memory_slice(entry, sizeof(mlk_poly)), object_whole(buf)) + invariant(ctr <= MLKEM_N) + invariant(array_bound(entry->coeffs, 0, ctr, 0, MLKEM_Q))) + { + mlk_xof_squeezeblocks(buf, 1, &state); + ctr = mlk_rej_uniform(entry->coeffs, MLKEM_N, ctr, buf, buflen); + } + + mlk_xof_release(&state); + + /* Specification: Partially implements + * @[FIPS203, Section 3.3, Destruction of intermediate values] */ + mlk_zeroize(buf, sizeof(buf)); +} + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 2 || \ + !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +/** + * Load 4 bytes into a 32-bit integer in little-endian order. + * + * @reference{`load32_littleendian()` in the reference implementation @[REF].} + * + * @param[in] x Input byte array. + * + * @return 32-bit unsigned integer loaded from @p x. + */ +static uint32_t mlk_load32_littleendian(const uint8_t x[4]) +{ + uint32_t r; + r = (uint32_t)x[0]; + r |= (uint32_t)x[1] << 8; + r |= (uint32_t)x[2] << 16; + r |= (uint32_t)x[3] << 24; + return r; +} + +/* Reference: `cbd2()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_poly_cbd2(mlk_poly *r, const uint8_t buf[2 * MLKEM_N / 4]) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 8; i++) + __loop__( + invariant(i <= MLKEM_N / 8) + invariant(array_abs_bound(r->coeffs, 0, 8 * i, 3)) + decreases(MLKEM_N / 8 - i)) + { + unsigned j; + uint32_t t = mlk_load32_littleendian(buf + 4 * i); + uint32_t d = t & 0x55555555; + d += (t >> 1) & 0x55555555; + + for (j = 0; j < 8; j++) + __loop__( + invariant(i <= MLKEM_N / 8 && j <= 8) + invariant(array_abs_bound(r->coeffs, 0, 8 * i + j, 3)) + decreases(8 - j)) + { + /* Safety: The & 0x3 masks each value to 2 bits (range [0, 3]), so the + * truncation and subsequent subtraction in int16_t is lossless. */ + const int16_t a = (int16_t)((d >> (4 * j + 0)) & 0x3); + const int16_t b = (int16_t)((d >> (4 * j + 2)) & 0x3); + r->coeffs[8 * i + j] = (int16_t)(a - b); + } + } +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 2 || \ + !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 3 +/** + * Load 3 bytes into a 32-bit integer in little-endian order. + * + * This function is only needed for ML-KEM-512. + * + * @reference{`load24_littleendian()` in the reference implementation @[REF].} + * + * @param[in] x Input byte array. + * + * @return 32-bit unsigned integer loaded from @p x (most significant byte + * is zero). + */ +static uint32_t mlk_load24_littleendian(const uint8_t x[3]) +{ + uint32_t r; + r = (uint32_t)x[0]; + r |= (uint32_t)x[1] << 8; + r |= (uint32_t)x[2] << 16; + return r; +} + +/* Reference: `cbd3()` in the reference implementation @[REF]. */ +MLK_INTERNAL_API +void mlk_poly_cbd3(mlk_poly *r, const uint8_t buf[3 * MLKEM_N / 4]) +{ + unsigned i; + for (i = 0; i < MLKEM_N / 4; i++) + __loop__( + invariant(i <= MLKEM_N / 4) + invariant(array_abs_bound(r->coeffs, 0, 4 * i, 4)) + decreases(MLKEM_N / 4 - i)) + { + unsigned j; + const uint32_t t = mlk_load24_littleendian(buf + 3 * i); + uint32_t d = t & 0x00249249; + d += (t >> 1) & 0x00249249; + d += (t >> 2) & 0x00249249; + + for (j = 0; j < 4; j++) + __loop__( + invariant(i <= MLKEM_N / 4 && j <= 4) + invariant(array_abs_bound(r->coeffs, 0, 4 * i + j, 4)) + decreases(4 - j)) + { + /* Safety: The & 0x7 masks each value to 3 bits (range [0, 7]), so the + * truncation and subsequent subtraction in int16_t is lossless. */ + const int16_t a = (int16_t)((d >> (6 * j + 0)) & 0x7); + const int16_t b = (int16_t)((d >> (6 * j + 3)) & 0x7); + r->coeffs[4 * i + j] = (int16_t)(a - b); + } + } +} +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 3 */ + +#else /* !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(sampling) + +#endif /* MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +/* To facilitate single-compilation-unit (SCU) builds, undefine all macros. + * Don't modify by hand -- this is auto-generated by scripts/autogen. */ +#undef MLKEM_GEN_MATRIX_NBLOCKS diff --git a/src/mlkem-native/mlkem/src/sampling.h b/src/mlkem-native/mlkem/src/sampling.h new file mode 100644 index 0000000000..24462aeb03 --- /dev/null +++ b/src/mlkem-native/mlkem/src/sampling.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_SAMPLING_H +#define MLK_SAMPLING_H + +#include "cbmc.h" +#include "common.h" +#include "poly.h" + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 2 || \ + !defined(MLK_CONFIG_NO_ENCAPS_API) || !defined(MLK_CONFIG_NO_DECAPS_API) +#define mlk_poly_cbd2 MLK_NAMESPACE(poly_cbd2) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter eta=2. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_2].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +MLK_INTERNAL_API +void mlk_poly_cbd2(mlk_poly *r, const uint8_t buf[2 * MLKEM_N / 4]); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 2 || \ + !MLK_CONFIG_NO_ENCAPS_API || !MLK_CONFIG_NO_DECAPS_API */ + +#if defined(MLK_CONFIG_MULTILEVEL_WITH_SHARED) || MLKEM_ETA1 == 3 +#define mlk_poly_cbd3 MLK_NAMESPACE(poly_cbd3) +/** + * Given an array of uniformly random bytes, compute a polynomial with + * coefficients distributed according to a centered binomial distribution + * with parameter eta=3. + * + * This function is only needed for ML-KEM-512. + * + * @spec{Implements @[FIPS203, Algorithm 8, SamplePolyCBD_3].} + * + * @param[out] r Output polynomial. + * @param[in] buf Input byte array. + */ +MLK_INTERNAL_API +void mlk_poly_cbd3(mlk_poly *r, const uint8_t buf[3 * MLKEM_N / 4]); +#endif /* MLK_CONFIG_MULTILEVEL_WITH_SHARED || MLKEM_ETA1 == 3 */ + +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +#define mlk_poly_rej_uniform_x4 MLK_NAMESPACE(poly_rej_uniform_x4) +/** + * Generate four polynomials using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. + * + * @spec{Implements @[FIPS203, Algorithm 7, SampleNTT].} + * + * @param[out] vec0 Polynomial to be sampled. + * @param[out] vec1 Polynomial to be sampled. + * @param[out] vec2 Polynomial to be sampled. + * @param[out] vec3 Polynomial to be sampled. + * @param[in] seed Consecutive array of 4 seed buffers of size + * MLKEM_SYMBYTES + 2 each, plus padding for alignment. + */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform_x4(mlk_poly *vec0, mlk_poly *vec1, mlk_poly *vec2, + mlk_poly *vec3, + uint8_t seed[4][MLK_ALIGN_UP(MLKEM_SYMBYTES + 2)]) +__contract__( + requires(memory_no_alias(vec0, sizeof(mlk_poly))) + requires(memory_no_alias(vec1, sizeof(mlk_poly))) + requires(memory_no_alias(vec2, sizeof(mlk_poly))) + requires(memory_no_alias(vec3, sizeof(mlk_poly))) + requires(memory_no_alias(seed, 4 * MLK_ALIGN_UP(MLKEM_SYMBYTES + 2))) + assigns(memory_slice(vec0, sizeof(mlk_poly))) + assigns(memory_slice(vec1, sizeof(mlk_poly))) + assigns(memory_slice(vec2, sizeof(mlk_poly))) + assigns(memory_slice(vec3, sizeof(mlk_poly))) + ensures(array_bound(vec0->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec1->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec2->coeffs, 0, MLKEM_N, 0, MLKEM_Q)) + ensures(array_bound(vec3->coeffs, 0, MLKEM_N, 0, MLKEM_Q))); +#endif /* !MLK_CONFIG_SERIAL_FIPS202_ONLY */ + +#define mlk_poly_rej_uniform MLK_NAMESPACE(poly_rej_uniform) +/** + * Generate a polynomial using rejection sampling on (pseudo-)uniformly + * random bytes sampled from a seed. + * + * @spec{Implements @[FIPS203, Algorithm 7, SampleNTT].} + * + * @param[out] entry Polynomial to be sampled. + * @param[in] seed Seed buffer of size MLKEM_SYMBYTES + 2. + */ +MLK_INTERNAL_API +void mlk_poly_rej_uniform(mlk_poly *entry, uint8_t seed[MLKEM_SYMBYTES + 2]) +__contract__( + requires(memory_no_alias(entry, sizeof(mlk_poly))) + requires(memory_no_alias(seed, MLKEM_SYMBYTES + 2)) + assigns(memory_slice(entry, sizeof(mlk_poly))) + ensures(array_bound(entry->coeffs, 0, MLKEM_N, 0, MLKEM_Q))); + +#endif /* !MLK_SAMPLING_H */ diff --git a/src/mlkem-native/mlkem/src/symmetric.h b/src/mlkem-native/mlkem/src/symmetric.h new file mode 100644 index 0000000000..68d7e1a0cd --- /dev/null +++ b/src/mlkem-native/mlkem/src/symmetric.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + */ + +#ifndef MLK_SYMMETRIC_H +#define MLK_SYMMETRIC_H + + +#include "cbmc.h" +#include "common.h" +#include MLK_FIPS202_HEADER_FILE +#if !defined(MLK_CONFIG_SERIAL_FIPS202_ONLY) +#include MLK_FIPS202X4_HEADER_FILE +#endif + +/* Macros denoting FIPS 203 specific Hash functions */ + +/* Hash function H, @[FIPS203, Section 4.1, Eq (4.4)] */ +#define mlk_hash_h(OUT, IN, INBYTES) mlk_sha3_256(OUT, IN, INBYTES) + +/* Hash function G, @[FIPS203, Section 4.1, Eq (4.5)] */ +#define mlk_hash_g(OUT, IN, INBYTES) mlk_sha3_512(OUT, IN, INBYTES) + +/* Hash function J, @[FIPS203, Section 4.1, Eq (4.4)] */ +#define mlk_hash_j(OUT, IN, INBYTES) \ + mlk_shake256(OUT, MLKEM_SYMBYTES, IN, INBYTES) + +/* PRF function, @[FIPS203, Section 4.1, Eq (4.3)] + * Referring to (eq 4.3), `OUT` is assumed to contain `s || b`. */ +#define mlk_prf_eta(ETA, OUT, IN) \ + mlk_shake256(OUT, (ETA) * MLKEM_N / 4, IN, MLKEM_SYMBYTES + 1) +#define mlk_prf_eta1(OUT, IN) mlk_prf_eta(MLKEM_ETA1, OUT, IN) +#define mlk_prf_eta2(OUT, IN) mlk_prf_eta(MLKEM_ETA2, OUT, IN) +#define mlk_prf_eta1_x4(OUT, IN) \ + mlk_shake256x4((OUT)[0], (OUT)[1], (OUT)[2], (OUT)[3], \ + (MLKEM_ETA1 * MLKEM_N / 4), (IN)[0], (IN)[1], (IN)[2], \ + (IN)[3], MLKEM_SYMBYTES + 1) + +/* XOF function, FIPS 203 4.1 */ +#define mlk_xof_ctx mlk_shake128ctx +#define mlk_xof_x4_ctx mlk_shake128x4ctx +#define mlk_xof_init(CTX) mlk_shake128_init((CTX)) +#define mlk_xof_absorb(CTX, IN, INBYTES) \ + mlk_shake128_absorb_once((CTX), (IN), (INBYTES)) +#define mlk_xof_squeezeblocks(BUF, NBLOCKS, CTX) \ + mlk_shake128_squeezeblocks((BUF), (NBLOCKS), (CTX)) +#define mlk_xof_release(CTX) mlk_shake128_release((CTX)) + +#define mlk_xof_x4_init(CTX) mlk_shake128x4_init((CTX)) +#define mlk_xof_x4_absorb(CTX, IN, INBYTES) \ + mlk_shake128x4_absorb_once((CTX), (IN)[0], (IN)[1], (IN)[2], (IN)[3], \ + (INBYTES)) +#define mlk_xof_x4_squeezeblocks(BUF, NBLOCKS, CTX) \ + mlk_shake128x4_squeezeblocks((BUF)[0], (BUF)[1], (BUF)[2], (BUF)[3], \ + (NBLOCKS), (CTX)) +#define mlk_xof_x4_release(CTX) mlk_shake128x4_release((CTX)) + +#define MLK_XOF_RATE SHAKE128_RATE + +#endif /* !MLK_SYMMETRIC_H */ diff --git a/src/mlkem-native/mlkem/src/sys.h b/src/mlkem-native/mlkem/src/sys.h new file mode 100644 index 0000000000..c430f77b0d --- /dev/null +++ b/src/mlkem-native/mlkem/src/sys.h @@ -0,0 +1,320 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLK_SYS_H +#define MLK_SYS_H + +#if !defined(MLK_CONFIG_NO_ASM) && (defined(__GNUC__) || defined(__clang__)) +#define MLK_HAVE_INLINE_ASM +#endif + +/* Try to find endianness, if not forced through CFLAGS already */ +#if !defined(MLK_SYS_LITTLE_ENDIAN) && !defined(MLK_SYS_BIG_ENDIAN) +#if defined(__BYTE_ORDER__) +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define MLK_SYS_LITTLE_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MLK_SYS_BIG_ENDIAN +#else +#error "__BYTE_ORDER__ defined, but don't recognize value." +#endif +#endif /* __BYTE_ORDER__ */ + +/* MSVC does not define __BYTE_ORDER__. However, MSVC only supports + * little endian x86, x86_64, and AArch64. It is, hence, safe to assume + * little endian. */ +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_AMD64) || \ + defined(_M_IX86) || defined(_M_ARM64)) +#define MLK_SYS_LITTLE_ENDIAN +#endif + +#endif /* !MLK_SYS_LITTLE_ENDIAN && !MLK_SYS_BIG_ENDIAN */ + +/* Check if we're running on an AArch64 little endian system. _M_ARM64 is set by + * MSVC. */ +#if defined(__AARCH64EL__) || defined(_M_ARM64) +#define MLK_SYS_AARCH64 +#endif + +/* Check if the AArch64 compilation target supports NEON (Advanced SIMD). + * + * Some compilers also define __ARM_NEON__, but __ARM_NEON is the most reliable + * signal. Specifically, clang on Apple appears to keep __ARM_NEON__ set even if + * -march=armv8-a+nosimd is set. + * + * gcc 4.8 -- the first gcc version introducing Neon support -- sets neither + * __ARM_NEON nor __ARM_NEON__; in fact, there is no preprocessor signal that + * Neon is enabled. If you use gcc 4.8, you should set __ARM_NEON manually. + * gcc 4.9 onwards do set __ARM_NEON. + */ +#if defined(MLK_SYS_AARCH64) && defined(__ARM_NEON) +#define MLK_SYS_AARCH64_NEON +#endif + +/* Check if we're running on an AArch64 big endian system. */ +#if defined(__AARCH64EB__) +#define MLK_SYS_AARCH64_EB +#endif + +/* Check if we're running on an Armv8.1-M system with MVE */ +#if defined(__ARM_ARCH_8_1M_MAIN__) || defined(__ARM_FEATURE_MVE) +#define MLK_SYS_ARMV81M_MVE +#endif + +/* Check if we're running on an x86_64 system. */ +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) +#define MLK_SYS_X86_64 +#if defined(__AVX2__) +#define MLK_SYS_X86_64_AVX2 +#endif +#endif /* __x86_64__ || _M_X64 || _M_AMD64 */ + +#if defined(MLK_SYS_LITTLE_ENDIAN) && defined(__powerpc64__) +#define MLK_SYS_PPC64LE +#endif + +#if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 64 +#define MLK_SYS_RISCV64 +#endif + +#if defined(MLK_SYS_RISCV64) && defined(__riscv_vector) && \ + defined(__riscv_v_intrinsic) +#define MLK_SYS_RISCV64_RVV +#endif + +#if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 32 +#define MLK_SYS_RISCV32 +#endif + +#if defined(_WIN32) +#define MLK_SYS_WINDOWS +#endif + +#if defined(__linux__) +#define MLK_SYS_LINUX +#endif + +#if defined(__APPLE__) +#define MLK_SYS_APPLE +#endif + +#if defined(MLK_FORCE_AARCH64) && !defined(MLK_SYS_AARCH64) +#error "MLK_FORCE_AARCH64 is set, but we don't seem to be on an AArch64 system." +#endif + +#if defined(MLK_FORCE_AARCH64_EB) && !defined(MLK_SYS_AARCH64_EB) +#error \ + "MLK_FORCE_AARCH64_EB is set, but we don't seem to be on an AArch64 system." +#endif + +#if defined(MLK_FORCE_X86_64) && !defined(MLK_SYS_X86_64) +#error "MLK_FORCE_X86_64 is set, but we don't seem to be on an X86_64 system." +#endif + +#if defined(MLK_FORCE_PPC64LE) && !defined(MLK_SYS_PPC64LE) +#error "MLK_FORCE_PPC64LE is set, but we don't seem to be on a PPC64LE system." +#endif + +#if defined(MLK_FORCE_RISCV64) && !defined(MLK_SYS_RISCV64) +#error "MLK_FORCE_RISCV64 is set, but we don't seem to be on a RISCV64 system." +#endif + +#if defined(MLK_FORCE_RISCV32) && !defined(MLK_SYS_RISCV32) +#error "MLK_FORCE_RISCV32 is set, but we don't seem to be on a RISCV32 system." +#endif + +/* + * MLK_INLINE: Hint for inlining. + * - MSVC: __inline + * - C99+: inline + * - GCC/Clang C90: __attribute__((unused)) to silence warnings + * - Other C90: empty + */ +#if !defined(MLK_INLINE) +#if defined(_MSC_VER) +#define MLK_INLINE __inline +#elif defined(inline) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +#define MLK_INLINE inline +#elif defined(__GNUC__) || defined(__clang__) +#define MLK_INLINE __attribute__((unused)) +#else +#define MLK_INLINE +#endif +#endif /* !MLK_INLINE */ + +/* + * MLK_ALWAYS_INLINE: Force inlining. + * - MSVC: __forceinline + * - GCC/Clang C99+: MLK_INLINE __attribute__((always_inline)) + * - Other: MLK_INLINE (no forced inlining) + */ +#if !defined(MLK_ALWAYS_INLINE) +#if defined(_MSC_VER) +#define MLK_ALWAYS_INLINE __forceinline +#elif (defined(__GNUC__) || defined(__clang__)) && \ + (defined(inline) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L)) +#define MLK_ALWAYS_INLINE MLK_INLINE __attribute__((always_inline)) +#else +#define MLK_ALWAYS_INLINE MLK_INLINE +#endif +#endif /* !MLK_ALWAYS_INLINE */ + +/* + * MLK_NOINLINE: Prevent inlining. + * - MSVC: __declspec(noinline) + * - GCC/Clang: __attribute__((noinline)) + * - Other: empty + */ +#if !defined(MLK_NOINLINE) +#if defined(_MSC_VER) +#define MLK_NOINLINE __declspec(noinline) +#elif defined(__GNUC__) || defined(__clang__) +#define MLK_NOINLINE __attribute__((noinline)) +#else +#define MLK_NOINLINE +#endif +#endif /* !MLK_NOINLINE */ + +#ifndef MLK_STATIC_TESTABLE +#define MLK_STATIC_TESTABLE static +#endif + +/* + * C90 does not have the restrict compiler directive yet. + * We don't use it in C90 builds. + */ +#if !defined(restrict) +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define MLK_RESTRICT restrict +#else +#define MLK_RESTRICT +#endif + +#else /* !restrict */ + +#define MLK_RESTRICT restrict +#endif /* restrict */ + +#define MLK_DEFAULT_ALIGN 32 +#define MLK_ALIGN_UP(N) \ + ((((N) + (MLK_DEFAULT_ALIGN - 1)) / MLK_DEFAULT_ALIGN) * MLK_DEFAULT_ALIGN) +#if defined(__GNUC__) +#define MLK_ALIGN __attribute__((aligned(MLK_DEFAULT_ALIGN))) +#elif defined(_MSC_VER) +#define MLK_ALIGN __declspec(align(MLK_DEFAULT_ALIGN)) +#else +#define MLK_ALIGN /* No known support for alignment constraints */ +#endif + + +/* New X86_64 CPUs support control-flow protection using the CET instructions. + * When enabled (through -fcf-protection=), all compilation units (including + * empty ones) need to support CET for this to work. + * For assembly, this means that source files need to signal support for + * CET by setting the appropriate note.gnu.property section. + * This can be achieved by including the header in all assembly file. + * This file also provides the _CET_ENDBR macro which needs to be placed at + * every potential target of an indirect branch. + * If CET is enabled _CET_ENDBR maps to the endbr64 instruction, otherwise + * it is empty. + * In case the compiler does not support CET (e.g., +#define MLK_CET_ENDBR _CET_ENDBR +#else +#define MLK_CET_ENDBR +#endif +#endif /* MLK_SYS_X86_64 */ + +#if defined(MLK_CONFIG_CT_TESTING_ENABLED) && !defined(__ASSEMBLER__) +#include +#define MLK_CT_TESTING_SECRET(ptr, len) \ + VALGRIND_MAKE_MEM_UNDEFINED((ptr), (len)) +#define MLK_CT_TESTING_DECLASSIFY(ptr, len) \ + VALGRIND_MAKE_MEM_DEFINED((ptr), (len)) +#else /* MLK_CONFIG_CT_TESTING_ENABLED && !__ASSEMBLER__ */ +#define MLK_CT_TESTING_SECRET(ptr, len) \ + do \ + { \ + } while (0) +#define MLK_CT_TESTING_DECLASSIFY(ptr, len) \ + do \ + { \ + } while (0) +#endif /* !(MLK_CONFIG_CT_TESTING_ENABLED && !__ASSEMBLER__) */ + +#if defined(__GNUC__) || defined(__clang__) +#define MLK_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result)) +#else +#define MLK_MUST_CHECK_RETURN_VALUE +#endif + +/* The x86_64 assembly backend uses the SysV calling convention. On Windows, + * where the Microsoft x64 calling convention is the default, it can still be + * used with compilers that allow choosing the calling convention per + * function: GCC and Clang support __attribute__((sysv_abi)), which makes + * calls to the annotated function follow the SysV calling convention. + * + * MLK_SYSV_ABI_SUPPORTED signals that the toolchain can call SysV assembly + * routines; the x86_64 assembly backend is only enabled if it is defined. + * MLK_SYSV_ABI is the attribute carried by declarations of x86_64 assembly + * routines. Both macros can be set externally for toolchains offering an + * equivalent mechanism that is not recognized here. */ +#if defined(MLK_SYS_X86_64) && !defined(MLK_SYSV_ABI_SUPPORTED) +#if !defined(MLK_SYS_WINDOWS) || defined(__GNUC__) || defined(__clang__) +#define MLK_SYSV_ABI_SUPPORTED +#endif +#endif + +#if !defined(MLK_SYSV_ABI) +#if defined(MLK_SYS_WINDOWS) && defined(MLK_SYSV_ABI_SUPPORTED) +#define MLK_SYSV_ABI __attribute__((sysv_abi)) +#else +#define MLK_SYSV_ABI +#endif +#endif /* !MLK_SYSV_ABI */ + +#if !defined(__ASSEMBLER__) +/* System capability enumeration */ +typedef enum +{ + /* x86_64 */ + MLK_SYS_CAP_X86_64_AVX2, + /* AArch64 */ + MLK_SYS_CAP_AARCH64_NEON, + MLK_SYS_CAP_AARCH64_SHA3, + /* Armv8.1-M */ + MLK_SYS_CAP_ARMV81M_MVE +} mlk_sys_cap; + +#if !defined(MLK_CONFIG_CUSTOM_CAPABILITY_FUNC) +#include "cbmc.h" + +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) +__contract__( + ensures(return_value == 0 || return_value == 1) +) +{ + /* By default, we rely on compile-time feature detection/specification: + * If a feature is enabled at compile-time, we assume it is supported by + * the host that the resulting library/binary will be built on. + * If this assumption is not true, you MUST overwrite this function. + * See the documentation of MLK_CONFIG_CUSTOM_CAPABILITY_FUNC in + * mlkem_native_config.h for more information. */ + (void)cap; + return 1; +} +#endif /* !MLK_CONFIG_CUSTOM_CAPABILITY_FUNC */ +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLK_SYS_H */ diff --git a/src/mlkem-native/mlkem/src/verify.c b/src/mlkem-native/mlkem/src/verify.c new file mode 100644 index 0000000000..db760f7271 --- /dev/null +++ b/src/mlkem-native/mlkem/src/verify.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#include "verify.h" + +#if !defined(MLK_USE_ASM_VALUE_BARRIER) && \ + !defined(MLK_CONFIG_MULTILEVEL_NO_SHARED) +/* + * Masking value used in constant-time functions from + * verify.h to block the compiler's range analysis and + * thereby reduce the risk of compiler-introduced branches. + */ +volatile uint64_t mlk_ct_opt_blocker_u64 = 0; + +#else /* !MLK_USE_ASM_VALUE_BARRIER && !MLK_CONFIG_MULTILEVEL_NO_SHARED */ + +MLK_EMPTY_CU(verify) + +#endif /* !(!MLK_USE_ASM_VALUE_BARRIER && !MLK_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/src/mlkem-native/mlkem/src/verify.h b/src/mlkem-native/mlkem/src/verify.h new file mode 100644 index 0000000000..8302a1c41b --- /dev/null +++ b/src/mlkem-native/mlkem/src/verify.h @@ -0,0 +1,447 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [FIPS203] + * FIPS 203 Module-Lattice-Based Key-Encapsulation Mechanism Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/203/final + * + * - [REF] + * CRYSTALS-Kyber C reference implementation + * Bos, Ducas, Kiltz, Lepoint, Lyubashevsky, Schanck, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/kyber/tree/main/ref + * + * - [libmceliece] + * libmceliece implementation of Classic McEliece + * Bernstein, Chou + * https://lib.mceliece.org/ + * + * - [optblocker] + * PQC forum post on opt-blockers using volatile globals + * Daniel J. Bernstein + * https://groups.google.com/a/list.nist.gov/g/pqc-forum/c/hqbtIGFKIpU/m/H14H0wOlBgAJ + */ + +#ifndef MLK_VERIFY_H +#define MLK_VERIFY_H + + +#include "cbmc.h" +#include "common.h" + +/* Constant-time comparisons and conditional operations + + We reduce the risk for compilation into variable-time code + through the use of 'value barriers'. + + Functionally, a value barrier is a no-op. To the compiler, however, + it constitutes an arbitrary modification of its input, and therefore + harden's value propagation and range analysis. + + We consider two approaches to implement a value barrier: + - An empty inline asm block which marks the target value as clobbered. + - XOR'ing with the value of a volatile global that's set to 0; + see @[optblocker] for a discussion of this idea, and + @[libmceliece, inttypes/crypto_intN.h] for an implementation. + + The first approach is cheap because it only prevents the compiler + from reasoning about the value of the variable past the barrier, + but does not directly generate additional instructions. + + The second approach generates redundant loads and XOR operations + and therefore comes at a higher runtime cost. However, it appears + more robust towards optimization, as compilers should never drop + a volatile load. + + We use the empty-ASM value barrier for GCC and clang, and fall + back to the global volatile barrier otherwise. + + The global value barrier can be forced by setting + MLK_CONFIG_NO_ASM_VALUE_BARRIER. + +*/ + +#if defined(MLK_HAVE_INLINE_ASM) && !defined(MLK_CONFIG_NO_ASM_VALUE_BARRIER) +#define MLK_USE_ASM_VALUE_BARRIER +#endif + +#if !defined(MLK_USE_ASM_VALUE_BARRIER) + +/* + * Declaration of global volatile that the global value barrier + * is loading from and masking with. + */ +#define mlk_ct_opt_blocker_u64 MLK_NAMESPACE(ct_opt_blocker_u64) +extern volatile uint64_t mlk_ct_opt_blocker_u64; + +/* Helper functions for obtaining global masks of various sizes */ + +/* This contract is not proved but treated as an axiom. + * + * Its validity relies on the assumption that the global opt-blocker + * constant mlk_ct_opt_blocker_u64 is not modified. + */ +static MLK_INLINE uint64_t mlk_ct_get_optblocker_u64(void) +__contract__(ensures(return_value == 0)) { return mlk_ct_opt_blocker_u64; } + +static MLK_INLINE uint8_t mlk_ct_get_optblocker_u8(void) +__contract__(ensures(return_value == 0)) { return (uint8_t)mlk_ct_get_optblocker_u64(); } + +static MLK_INLINE uint32_t mlk_ct_get_optblocker_u32(void) +__contract__(ensures(return_value == 0)) { return (uint32_t)mlk_ct_get_optblocker_u64(); } + +static MLK_INLINE int32_t mlk_ct_get_optblocker_i32(void) +__contract__(ensures(return_value == 0)) { return (int32_t)mlk_ct_get_optblocker_u64(); } + +/* Opt-blocker based implementation of value barriers */ +static MLK_INLINE uint32_t mlk_value_barrier_u32(uint32_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_u32()); } + +static MLK_INLINE int32_t mlk_value_barrier_i32(int32_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_i32()); } + +static MLK_INLINE uint8_t mlk_value_barrier_u8(uint8_t b) +__contract__(ensures(return_value == b)) { return (b ^ mlk_ct_get_optblocker_u8()); } + +#else /* !MLK_USE_ASM_VALUE_BARRIER */ + +static MLK_INLINE uint32_t mlk_value_barrier_u32(uint32_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +static MLK_INLINE int32_t mlk_value_barrier_i32(int32_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +static MLK_INLINE uint8_t mlk_value_barrier_u8(uint8_t b) +__contract__(ensures(return_value == b)) +{ + __asm__ volatile("" : "+r"(b)); + return b; +} + +#endif /* MLK_USE_ASM_VALUE_BARRIER */ + +#ifdef CBMC +#pragma CPROVER check push +#pragma CPROVER check disable "conversion" +#endif +/** + * Cast uint16 value to int16. + * + * @param x Input value. + * + * @return For uint16_t x, the unique y in int16_t so that x == y mod 2^16. + * Concretely: + * - x < 32768: returns x + * - x >= 32768: returns x - 65536 + */ +static MLK_ALWAYS_INLINE int16_t mlk_cast_uint16_to_int16(uint16_t x) +{ + /* + * PORTABILITY: This relies on uint16_t -> int16_t + * being implemented as the inverse of int16_t -> uint16_t, + * which is implementation-defined (C99 6.3.1.3 (3)) + * CBMC (correctly) fails to prove this conversion is OK, + * so we have to suppress that check here + */ + return (int16_t)x; +} +#ifdef CBMC +#pragma CPROVER check pop +#endif + +/** + * Cast int32 value to uint16 as per C standard. + * + * @param x Input value. + * + * @return For int32_t x, the unique y in uint16_t so that x == y mod 2^16. + */ +static MLK_ALWAYS_INLINE uint16_t mlk_cast_int32_to_uint16(int32_t x) +{ + return (uint16_t)(x & (int32_t)UINT16_MAX); +} + +/** + * Cast int16 value to uint16 as per C standard. + * + * @param x Input value. + * + * @return For int16_t x, the unique y in uint16_t so that x == y mod 2^16. + */ +static MLK_ALWAYS_INLINE uint16_t mlk_cast_int16_to_uint16(int32_t x) +{ + return mlk_cast_int32_to_uint16(x); +} + +/** + * Return 0 if input is non-negative, and -1 otherwise. + * + * @reference{Embedded in the polynomial compression function in the + * reference implementation @[REF]. Used as part of signed->unsigned + * conversion for modular representatives to detect whether the input is + * negative. This happens in `mlk_poly_reduce()` here, and as part of + * polynomial compression functions in the reference implementation. See + * `mlk_poly_reduce()`. We use value barriers to reduce the risk of + * compiler-introduced branches.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFFFF). + */ +static MLK_INLINE uint16_t mlk_ct_cmask_neg_i16(int16_t x) +__contract__(ensures(return_value == ((x < 0) ? 0xFFFF : 0))) +{ + int32_t tmp = mlk_value_barrier_i32((int32_t)x); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + tmp >>= 16; + return mlk_cast_int32_to_uint16(tmp); +} + +/** + * Return 0 if input is zero, and -1 otherwise. + * + * @reference{Embedded in `cmov_int16()` in the reference implementation + * @[REF]. Uses a value barrier and shift instead of `b = -b` to convert + * condition into mask.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFFFF). + */ +static MLK_INLINE uint16_t mlk_ct_cmask_nonzero_u16(uint16_t x) +__contract__(ensures(return_value == ((x == 0) ? 0 : 0xFFFF))) +{ + int32_t tmp = mlk_value_barrier_i32(-((int32_t)x)); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ + tmp >>= 16; + return mlk_cast_int32_to_uint16(tmp); +} + +/** + * Return 0 if input is zero, and -1 otherwise. + * + * @reference{Embedded in `verify()` and `cmov()` in the reference + * implementation @[REF]. We include a value barrier not present in the + * reference implementation, to prevent the compiler from realizing that + * this function returns a mask.} + * + * @param x Value to be converted into a mask. + * + * @return Mask value (0 or 0xFF). + */ +static MLK_INLINE uint8_t mlk_ct_cmask_nonzero_u8(uint8_t x) +__contract__(ensures(return_value == ((x == 0) ? 0 : 0xFF))) +{ + uint16_t mask = mlk_ct_cmask_nonzero_u16((uint16_t)x); + return (uint8_t)(mask & 0xFF); +} + +/** + * Functionally equivalent to cond ? a : b, but implemented with guards + * against compiler-introduced branches. + * + * @spec{With `a = MLKEM_Q_HALF` and `b=0`, this essentially implements + * `Decompress_1` @[FIPS203, Eq (4.8)] in `mlk_poly_frommsg()`. With + * `a = x + MLKEM_Q`, `b = x`, and `cond` indicating whether `x` is negative, + * implements signed->unsigned conversion of modular representatives. + * Questions of representation are not considered in the specification + * @[FIPS203, Section 2.4.1, "The pseudocode is agnostic regarding how an + * integer modulo 𝑚 is represented in actual implementations"].} + * + * @reference{Embedded in the polynomial compression function in the + * reference implementation @[REF]. Used as part of signed->unsigned + * conversion for modular representatives. This happens in `mlk_poly_reduce()` + * here, and as part of polynomial compression functions in @[REF]. See + * `mlk_poly_reduce()`. Barrier to reduce the risk of compiler-introduced + * branches. For `a = MLKEM_Q_HALF` and `b=0`, also embedded in + * `poly_frommsg()` from the reference implementation, which uses + * `cmov_int16()` instead.} + * + * @param a First alternative. + * @param b Second alternative. + * @param cond Condition variable. + * + * @return @p a if @p cond != 0, else @p b. + */ +static MLK_INLINE int16_t mlk_ct_sel_int16(int16_t a, int16_t b, uint16_t cond) +__contract__(ensures(return_value == (cond ? a : b))) +{ + uint16_t au = mlk_cast_int16_to_uint16(a); + uint16_t bu = mlk_cast_int16_to_uint16(b); + uint16_t res = bu ^ (mlk_ct_cmask_nonzero_u16(cond) & (au ^ bu)); + return mlk_cast_uint16_to_int16(res); +} + +/** + * Functionally equivalent to cond ? a : b, but implemented with guards + * against compiler-introduced branches. + * + * @reference{Embedded into `cmov()` in the reference implementation @[REF]. + * Uses a value barrier to get mask from condition value.} + * + * @param a First alternative. + * @param b Second alternative. + * @param cond Condition variable. + * + * @return @p a if @p cond != 0, else @p b. + */ +static MLK_INLINE uint8_t mlk_ct_sel_uint8(uint8_t a, uint8_t b, uint8_t cond) +__contract__(ensures(return_value == (cond ? a : b))) +{ + return b ^ (mlk_ct_cmask_nonzero_u8(cond) & (a ^ b)); +} + +/** + * Compare two arrays for equality in constant time. + * + * @spec{Used to securely compute conditional move in @[FIPS203, Algorithm + * 18 (ML-KEM.Decaps_Internal, L9-11].} + * + * @reference{`cmov()` in the reference implementation @[REF]. We return + * `uint8_t`, not `int`. We use an additional XOR-accumulator in the + * comparison loop which prevents early abort if the OR-accumulator is 0xFF. + * We use a value barrier to convert the OR-accumulator into a mask; the + * reference implementation uses a shift which the compiler can argue to + * result in either 0 or 0xFF..FF.} + * + * @param[in] a First byte array. + * @param[in] b Second byte array. + * @param len Length of the byte arrays, upper-bounded to UINT16_MAX to + * control proof complexity only. + * + * @retval 0 The byte arrays are equal. + * @retval 0xFF The byte arrays are not equal. + */ +static MLK_INLINE uint8_t mlk_ct_memcmp(const uint8_t *a, const uint8_t *b, + const size_t len) +__contract__( + requires(len <= UINT16_MAX) + requires(memory_no_alias(a, len)) + requires(memory_no_alias(b, len)) + ensures((return_value == 0) || (return_value == 0xFF)) + ensures((return_value == 0) == forall(i, 0, len, (a[i] == b[i])))) +{ + uint8_t r = 0, s = 0; + unsigned i; + + for (i = 0; i < len; i++) + __loop__( + invariant(i <= len) + invariant((r == 0) == (forall(k, 0, i, (a[k] == b[k])))) + decreases(len - i)) + { + r |= a[i] ^ b[i]; + /* s is useless, but prevents the loop from being aborted once r=0xff. */ + s ^= a[i] ^ b[i]; + } + + /* + * - Convert r into a mask; this may not be necessary, but is an additional + * safeguard + * towards leaking information about a and b. + * - XOR twice with s, separated by a value barrier, to prevent the compile + * from dropping the s computation in the loop. + */ + return (mlk_value_barrier_u8(mlk_ct_cmask_nonzero_u8(r) ^ s) ^ s); +} + +/** + * Copy len bytes from x to r if b is zero; don't modify x if b is non-zero. + * Assumes two's complement representation of negative integers. Runs in + * constant time. + * + * @spec{Used to securely compute conditional move in @[FIPS203, Algorithm + * 18 (ML-KEM.Decaps_Internal, L9-11].} + * + * @reference{`cmov()` in the reference implementation @[REF]. We move if + * condition value is `0`, not `1`. We use `mlk_ct_sel_uint8` for + * constant-time selection.} + * + * @param[out] r Output byte array. + * @param[in] x Input byte array. + * @param len Number of bytes to be copied. + * @param b Condition value. + */ +static MLK_INLINE void mlk_ct_cmov_zero(uint8_t *r, const uint8_t *x, + size_t len, uint8_t b) +__contract__( + requires(len <= UINT32_MAX) + requires(memory_no_alias(r, len)) + requires(memory_no_alias(x, len)) + assigns(memory_slice(r, len)) + ensures(forall(i, 0, len, (r[i] == (b == 0 ? x[i] : old(r)[i]))))) +{ + size_t i; + for (i = 0; i < len; i++) + __loop__( + invariant(i <= len) + invariant(forall(k, 0, i, r[k] == (b == 0 ? x[k] : loop_entry(r)[k]))) + decreases(len - i)) + { + r[i] = mlk_ct_sel_uint8(r[i], x[i], b); + } +} + +/** + * Force-zeroize a buffer. + * + * @spec{Used to implement @[FIPS203, Section 3.3, Destruction of + * intermediate values].} + * + * @reference{Not present in the reference implementation @[REF].} + * + * @param[out] ptr Buffer to be zeroed. + * @param len Number of bytes to be zeroed. + */ +#if !defined(MLK_CONFIG_CUSTOM_ZEROIZE) +#if defined(MLK_SYS_WINDOWS) +#include +#elif !defined(MLK_HAVE_INLINE_ASM) +#error No plausibly-secure implementation of mlk_zeroize available. Please provide your own using MLK_CONFIG_CUSTOM_ZEROIZE. +#endif + +static MLK_INLINE void mlk_zeroize(void *ptr, size_t len) +__contract__( + requires(len <= UINT32_MAX) + requires(memory_no_alias(ptr, len)) + assigns(memory_slice(ptr, len)) + ensures(array_zeroized_u8((uint8_t *)ptr, len))) +{ +#if defined(MLK_SYS_WINDOWS) + SecureZeroMemory(ptr, len); +#else + mlk_memset(ptr, 0, len); + /* This follows OpenSSL and seems sufficient to prevent the compiler + * from optimizing away the memset. + * + * If there was a reliable way to detect availability of memset_s(), + * that would be preferred. */ + __asm__ volatile("" : : "r"(ptr) : "memory"); +#endif /* !MLK_SYS_WINDOWS */ +} +#endif /* !MLK_CONFIG_CUSTOM_ZEROIZE */ + +#endif /* !MLK_VERIFY_H */ diff --git a/src/mlkem-native/mlkem/src/zetas.inc b/src/mlkem-native/mlkem/src/zetas.inc new file mode 100644 index 0000000000..00316daf67 --- /dev/null +++ b/src/mlkem-native/mlkem/src/zetas.inc @@ -0,0 +1,30 @@ +/* + * Copyright (c) The mlkem-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mlkem-native repository. + * Do not modify it directly. + */ + + +/* + * Table of zeta values used in the reference NTT and inverse NTT. + * See autogen for details. + */ +static MLK_ALIGN const int16_t mlk_zetas[128] = { + -1044, -758, -359, -1517, 1493, 1422, 287, 202, -171, 622, 1577, + 182, 962, -1202, -1474, 1468, 573, -1325, 264, 383, -829, 1458, + -1602, -130, -681, 1017, 732, 608, -1542, 411, -205, -1571, 1223, + 652, -552, 1015, -1293, 1491, -282, -1544, 516, -8, -320, -666, + -1618, -1162, 126, 1469, -853, -90, -271, 830, 107, -1421, -247, + -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, -1103, 430, + 555, 843, -1251, 871, 1550, 105, 422, 587, 177, -235, -291, + -460, 1574, 1653, -246, 778, 1159, -147, -777, 1483, -602, 1119, + -1590, 644, -872, 349, 418, 329, -156, -75, 817, 1097, 603, + 610, 1322, -1285, -1465, 384, -1215, -136, 1218, -1335, -874, 220, + -1187, -1659, -1185, -1530, -1278, 794, -1510, -854, -870, 478, -108, + -308, 996, 991, 958, -1460, 1522, 1628, +}; diff --git a/test/lint/README.md b/test/lint/README.md index df1dfc3931..16493b0508 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -130,6 +130,22 @@ The check accepts the exact documented downstream lockfile patch for `RUSTSEC-2026-0204-crossbeam-epoch-0.9.20`. Any other subtree drift remains a failure. +mlkem-native-check.sh +===================== +Run this script from the root of the repository to verify offline that the +vendored `src/mlkem-native` tree is byte-identical to the mlkem-native release +pinned in `contrib/devtools/mlkem-native.pin`: + +``` +test/lint/mlkem-native-check.sh +``` + +It compares the git tree id of `src/mlkem-native/mlkem` and the blob id of +`src/mlkem-native/LICENSE` in `HEAD` with the pinned ids, and fails on any other +entry or uncommitted change under `src/mlkem-native`. The lint runner's +`subtree` lint runs it. For the update workflow, see +[`doc/subtrees/mlkem-native.md`](../../doc/subtrees/mlkem-native.md). + lint-libbitcoinpqc-vectors.py ============================= Run this script from the root of the repository to verify bounded30 SPHINCS+ diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py index 7e90b5cd80..9bb55ded2b 100755 --- a/test/lint/lint-locale-dependence.py +++ b/test/lint/lint-locale-dependence.py @@ -56,6 +56,7 @@ "src/leveldb/", "src/secp256k1/", "src/minisketch/", + "src/mlkem-native/", "src/tinyformat.h", ] diff --git a/test/lint/lint_ignore_dirs.py b/test/lint/lint_ignore_dirs.py index 34dc945b5d..c5144dc424 100644 --- a/test/lint/lint_ignore_dirs.py +++ b/test/lint/lint_ignore_dirs.py @@ -10,5 +10,6 @@ "src/minisketch/", "src/ipc/libmultiprocess/", "src/libbitcoinpqc/", + "src/mlkem-native/", "contrib/photon/src/vendor/", ] diff --git a/test/lint/mlkem-native-check.sh b/test/lint/mlkem-native-check.sh new file mode 100755 index 0000000000..15ae3cdb04 --- /dev/null +++ b/test/lint/mlkem-native-check.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +export LC_ALL=C +set -euo pipefail + +readonly PREFIX="src/mlkem-native" +readonly PIN_FILE="contrib/devtools/mlkem-native.pin" + +usage() { + cat <&2 + echo " Cause: ${cause}" >&2 + echo " Fix: ${fix}" >&2 + exit 1 +} + +pin_value() { + local key="$1" + local value + value="$(awk -F= -v key="${key}" '$1 == key { print substr($0, length(key) + 2); exit }' "${PIN_FILE}")" + if [[ -z "${value}" ]]; then + fail "${PIN_FILE} has no ${key}= entry" \ + "the pin file was edited by hand or truncated" \ + "restore ${PIN_FILE} from git, or regenerate it with contrib/devtools/update-mlkem-native.sh" + fi + printf '%s\n' "${value}" +} + +require_object_id() { + local key="$1" + local value="$2" + if [[ ! "${value}" =~ ^[0-9a-f]{40}$ ]]; then + fail "${PIN_FILE} ${key}=${value} is not a 40-digit object id" \ + "the pin file was edited by hand" \ + "restore ${PIN_FILE} from git, or regenerate it with contrib/devtools/update-mlkem-native.sh" + fi +} + +if [[ "${1-}" == "--help" || "${1-}" == "-h" ]]; then + usage + exit 0 +fi + +if [[ $# -ne 0 ]]; then + usage >&2 + exit 2 +fi + +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + fail "not inside a git work tree" \ + "the check compares git object ids" \ + "run it from the root of a qbit checkout" +fi +cd "$(git rev-parse --show-toplevel)" + +if [[ ! -f "${PIN_FILE}" ]]; then + fail "${PIN_FILE} not found" \ + "the pin file was deleted or the check runs outside the repository root" \ + "restore ${PIN_FILE} from git" +fi + +pinned_tag="$(pin_value tag)" +pinned_commit="$(pin_value commit)" +pinned_tree="$(pin_value mlkem_tree)" +pinned_license="$(pin_value license_blob)" +require_object_id commit "${pinned_commit}" +require_object_id mlkem_tree "${pinned_tree}" +require_object_id license_blob "${pinned_license}" + +echo "${PIN_FILE} pins mlkem-native ${pinned_tag} commit ${pinned_commit}" + +top_entries="$(git ls-tree --name-only "HEAD:${PREFIX}" 2>/dev/null || true)" +if [[ -z "${top_entries}" ]]; then + fail "${PREFIX} not found in HEAD" \ + "the vendored tree was removed or never committed" \ + "run contrib/devtools/update-mlkem-native.sh and commit the result" +fi +if [[ "${top_entries}" != $'LICENSE\nmlkem' ]]; then + fail "${PREFIX} in HEAD holds entries other than LICENSE and mlkem: $(printf '%s' "${top_entries}" | tr '\n' ' ')" \ + "a file was added next to the vendored tree" \ + "move qbit-owned files out of ${PREFIX} (ML-KEM glue lives in src/crypto/mlkem*)" +fi + +current_tree="$(git rev-parse "HEAD:${PREFIX}/mlkem")" +echo "${PREFIX}/mlkem in HEAD is tree ${current_tree}" +if [[ "${current_tree}" != "${pinned_tree}" ]]; then + git diff --stat "${pinned_tree}" "${current_tree}" >&2 2>/dev/null || true + fail "${PREFIX}/mlkem tree ${current_tree} differs from pinned tree ${pinned_tree}" \ + "a vendored file was edited, added, removed or changed mode, or the pin was changed without re-vendoring" \ + "never edit ${PREFIX}; restore it with contrib/devtools/update-mlkem-native.sh" +fi + +current_license="$(git rev-parse "HEAD:${PREFIX}/LICENSE")" +echo "${PREFIX}/LICENSE in HEAD is blob ${current_license}" +if [[ "${current_license}" != "${pinned_license}" ]]; then + fail "${PREFIX}/LICENSE blob ${current_license} differs from pinned blob ${pinned_license}" \ + "the vendored LICENSE was edited, or the pin was changed without re-vendoring" \ + "never edit ${PREFIX}; restore it with contrib/devtools/update-mlkem-native.sh" +fi + +dirty="$(git status --porcelain --untracked-files=all -- "${PREFIX}")" +if [[ -n "${dirty}" ]]; then + printf '%s\n' "${dirty}" >&2 + fail "uncommitted changes under ${PREFIX}" \ + "a vendored file was edited, added or removed in the work tree or index" \ + "discard them with: git checkout HEAD -- ${PREFIX} && git clean -fdx -- ${PREFIX}" +fi + +echo "GOOD" diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 3ea377f623..9c8083aa61 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -224,6 +224,8 @@ fn get_pathspecs_default_excludes() -> Vec { // qbit-specific vendored source. Public release snapshot branches may not // carry git-subtree metadata until the public upstream repository exists. "src/libbitcoinpqc", + // Vendored byte for byte and pinned by tree id; never reformatted. + "src/mlkem-native", "doc/release-notes/release-notes-*", // archived notes "contrib/photon/src/vendor/", // externally sourced vendored code ]) @@ -246,6 +248,10 @@ fn lint_subtree() -> LintResult { .status() .expect("command_error") .success(); + good &= Command::new("test/lint/mlkem-native-check.sh") + .status() + .expect("command_error") + .success(); if good { Ok(()) } else { @@ -697,6 +703,8 @@ fn lint_markdown() -> LintResult { let bin_name = "mlc"; let mut md_ignore_paths = get_subtrees(); md_ignore_paths.push("./doc/development/README_doxygen.md"); + // Upstream's READMEs link to files outside the vendored mlkem/ directory. + md_ignore_paths.push("src/mlkem-native"); let md_ignore_path_str = md_ignore_paths.join(","); let mut cmd = Command::new(bin_name); From 8473a4a305438bd74e353f0bbcf8a853860f2a49 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 12:41:49 -0400 Subject: [PATCH 02/20] crypto: share x86 feature primitives and detect usable AVX2 Move the XGETBV instruction out of sha256.cpp into compat/cpuid.h as XGetBV(), next to GetCPUID(). SHA256AutoDetect keeps its own selection logic unchanged; it calls the shared primitive only. Add compat/cpu_features.{h,cpp} with a standalone answer to "can this CPU and OS run AVX2 code": the supported leaves are queried first, XCR0 is read only once leaf 1 reports OSXSAVE and AVX, and AVX2 is usable only with XMM and YMM state enabled and the leaf 7 AVX2 bit set. Unlike the SHA256 logic it does not depend on SSE4 or SHA-NI. HasMlkemX86Native() adds the other extensions that mlkem-native's x86_64 assembly executes, found by auditing the assembled object: SSSE3, SSE4.1, POPCNT and BMI2. A virtual machine can show AVX2 while hiding those, and checking AVX2 alone would then end in SIGILL. Both checks detect once and cache the result, because the ML-KEM capability hook asks about a hundred times per operation. Non-x86 targets and compilers without report false. Part of #184 (part 3, #190, D2). Co-Authored-By: Claude Opus 5.5 --- cmake/bitcoin-build-config.h.in | 3 + cmake/introspection.cmake | 12 ++ src/compat/cpu_features.cpp | 53 ++++++++ src/compat/cpu_features.h | 83 +++++++++++++ src/compat/cpuid.h | 8 ++ src/crypto/sha256.cpp | 4 +- src/test/CMakeLists.txt | 1 + src/test/cpu_features_tests.cpp | 212 ++++++++++++++++++++++++++++++++ src/util/CMakeLists.txt | 1 + 9 files changed, 374 insertions(+), 3 deletions(-) create mode 100644 src/compat/cpu_features.cpp create mode 100644 src/compat/cpu_features.h create mode 100644 src/test/cpu_features_tests.cpp diff --git a/cmake/bitcoin-build-config.h.in b/cmake/bitcoin-build-config.h.in index cdc92893e7..965cb72bac 100644 --- a/cmake/bitcoin-build-config.h.in +++ b/cmake/bitcoin-build-config.h.in @@ -70,6 +70,9 @@ /* Define this symbol if you have malloc_info */ #cmakedefine HAVE_MALLOC_INFO 1 +/* Define this symbol if __builtin_cpu_supports links for the target */ +#cmakedefine HAVE_BUILTIN_CPU_SUPPORTS 1 + /* Define this symbol if you have mallopt with M_ARENA_MAX */ #cmakedefine HAVE_MALLOPT_ARENA_MAX 1 diff --git a/cmake/introspection.cmake b/cmake/introspection.cmake index d225763397..c15a0274f3 100644 --- a/cmake/introspection.cmake +++ b/cmake/introspection.cmake @@ -39,6 +39,18 @@ check_cxx_source_compiles(" " STRERROR_R_CHAR_P ) +# Check that __builtin_cpu_supports links for the target. It needs __cpu_model +# from the compiler's runtime library, which some cross toolchains (macOS from +# depends) don't provide. Tests use it only as an independent CPU check. +check_cxx_source_compiles(" + int main() + { + __builtin_cpu_init(); + return __builtin_cpu_supports(\"avx2\") ? 0 : 1; + } + " HAVE_BUILTIN_CPU_SUPPORTS +) + # Check for malloc_info (for memory statistics information in getmemoryinfo). check_cxx_symbol_exists(malloc_info "malloc.h" HAVE_MALLOC_INFO) diff --git a/src/compat/cpu_features.cpp b/src/compat/cpu_features.cpp new file mode 100644 index 0000000000..ffe249b670 --- /dev/null +++ b/src/compat/cpu_features.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include + +#include + +namespace cpu_features { + +X86Features DetectX86Features(CpuidFn cpuid, XGetBVFn xgetbv) noexcept +{ + X86Features f; + uint32_t a{0}, b{0}, c{0}, d{0}; + cpuid(0, 0, a, b, c, d); + f.max_basic_leaf = a; + if (f.max_basic_leaf >= 1) { + cpuid(1, 0, a, b, c, d); + f.leaf1_ecx = c; + } + if (f.max_basic_leaf >= 7) { + cpuid(7, 0, a, b, c, d); + f.leaf7_ebx = b; + } + // XGETBV faults unless the OS enabled it, which leaf 1 reports as OSXSAVE. + if ((f.leaf1_ecx & LEAF1_ECX_OSXSAVE) != 0 && (f.leaf1_ecx & LEAF1_ECX_AVX) != 0) { + f.xcr0 = xgetbv(0); + } + return f; +} + +X86Features DetectX86Features() noexcept +{ +#if defined(HAVE_GETCPUID) + return DetectX86Features(GetCPUID, XGetBV); +#else + return {}; +#endif +} + +bool HasUsableAVX2() noexcept +{ + static const bool usable{HasUsableAVX2(DetectX86Features())}; + return usable; +} + +bool HasMlkemX86Native() noexcept +{ + static const bool usable{HasMlkemX86Native(DetectX86Features())}; + return usable; +} + +} // namespace cpu_features diff --git a/src/compat/cpu_features.h b/src/compat/cpu_features.h new file mode 100644 index 0000000000..9855211bb9 --- /dev/null +++ b/src/compat/cpu_features.h @@ -0,0 +1,83 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_COMPAT_CPU_FEATURES_H +#define QBIT_COMPAT_CPU_FEATURES_H + +#include + +/** + * Standalone x86 feature detection for code that dispatches at run time. + * + * This deliberately shares only the CPUID and XGETBV primitives with + * SHA256AutoDetect, whose selection logic is tuned for SHA256 (it skips leaf 7 + * without SSE4 and drops AVX2 when SHA-NI is present). + */ +namespace cpu_features { + +inline constexpr uint32_t LEAF1_ECX_SSSE3{uint32_t{1} << 9}; +inline constexpr uint32_t LEAF1_ECX_SSE41{uint32_t{1} << 19}; +inline constexpr uint32_t LEAF1_ECX_POPCNT{uint32_t{1} << 23}; +inline constexpr uint32_t LEAF1_ECX_OSXSAVE{uint32_t{1} << 27}; +inline constexpr uint32_t LEAF1_ECX_AVX{uint32_t{1} << 28}; +inline constexpr uint32_t LEAF7_EBX_AVX2{uint32_t{1} << 5}; +inline constexpr uint32_t LEAF7_EBX_BMI2{uint32_t{1} << 8}; +/** XCR0 bits for XMM (SSE) and YMM (AVX) register state saved by the OS. */ +inline constexpr uint64_t XCR0_XMM_YMM{0x6}; + +struct X86Features { + uint32_t max_basic_leaf{0}; + uint32_t leaf1_ecx{0}; + uint32_t leaf7_ebx{0}; + uint64_t xcr0{0}; +}; + +using CpuidFn = void (*)(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d); +using XGetBVFn = uint64_t (*)(uint32_t index); + +/** + * Query the supported leaves through the given primitives, then read XCR0 + * only if leaf 1 reports both OSXSAVE and AVX. Exposed so tests can supply + * synthetic primitives. + */ +X86Features DetectX86Features(CpuidFn cpuid, XGetBVFn xgetbv) noexcept; + +/** Query this CPU and OS. All fields are zero on non-x86 targets and unsupported compilers. */ +X86Features DetectX86Features() noexcept; + +/** Whether the CPU supports AVX2 and the OS saves YMM state, so AVX2 code can run. */ +constexpr bool HasUsableAVX2(const X86Features& f) noexcept +{ + return f.max_basic_leaf >= 7 && + (f.leaf1_ecx & LEAF1_ECX_OSXSAVE) != 0 && + (f.leaf1_ecx & LEAF1_ECX_AVX) != 0 && + (f.xcr0 & XCR0_XMM_YMM) == XCR0_XMM_YMM && + (f.leaf7_ebx & LEAF7_EBX_AVX2) != 0; +} + +/** HasUsableAVX2() for this CPU and OS, detected on first call and then cached. */ +bool HasUsableAVX2() noexcept; + +/** + * Whether this CPU and OS can run mlkem-native's x86_64 assembly. Besides + * AVX2 (with AVX and the OS saving YMM state), that assembly executes SSSE3, + * SSE4.1, POPCNT and BMI2 instructions, which a CPUID mask in a virtual + * machine can hide even when AVX2 is shown. The list comes from an audit of + * the assembled objects; doc/subtrees/mlkem-native.md says how to repeat it. + */ +constexpr bool HasMlkemX86Native(const X86Features& f) noexcept +{ + return HasUsableAVX2(f) && + (f.leaf1_ecx & LEAF1_ECX_SSSE3) != 0 && + (f.leaf1_ecx & LEAF1_ECX_SSE41) != 0 && + (f.leaf1_ecx & LEAF1_ECX_POPCNT) != 0 && + (f.leaf7_ebx & LEAF7_EBX_BMI2) != 0; +} + +/** HasMlkemX86Native() for this CPU and OS, detected on first call and then cached. */ +bool HasMlkemX86Native() noexcept; + +} // namespace cpu_features + +#endif // QBIT_COMPAT_CPU_FEATURES_H diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h index 10a959df75..562e1d8b5a 100644 --- a/src/compat/cpuid.h +++ b/src/compat/cpuid.h @@ -22,5 +22,13 @@ void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32 #endif } +/** Read extended control register `index` (XCR0 for 0). Faults unless CPUID leaf 1 reports OSXSAVE. */ +static inline uint64_t XGetBV(uint32_t index) noexcept +{ + uint32_t a, d; + __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(index)); + return (uint64_t{d} << 32) | a; +} + #endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) #endif // QBIT_COMPAT_CPUID_H diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index d75d01b6b7..9763d8bc10 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -573,9 +573,7 @@ bool SelfTest() { /** Check whether the OS has enabled AVX registers. */ bool AVXEnabled() { - uint32_t a, d; - __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(0)); - return (a & 6) == 6; + return (XGetBV(0) & 6) == 6; } #endif #endif // DISABLE_OPTIMIZED_SHA256 diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index a7ff86bb89..b08c3d0f2e 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -40,6 +40,7 @@ add_executable(test_bitcoin common_url_tests.cpp compilerbug_tests.cpp compress_tests.cpp + cpu_features_tests.cpp crypto_tests.cpp cuckoocache_tests.cpp dbwrapper_tests.cpp diff --git a/src/test/cpu_features_tests.cpp b/src/test/cpu_features_tests.cpp new file mode 100644 index 0000000000..ca9e3effbe --- /dev/null +++ b/src/test/cpu_features_tests.cpp @@ -0,0 +1,212 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include + +using namespace cpu_features; + +namespace { + +constexpr uint32_t LEAF7_EBX_SHA{uint32_t{1} << 29}; + +constexpr X86Features AVX2_USABLE{ + .max_basic_leaf = 7, + .leaf1_ecx = LEAF1_ECX_OSXSAVE | LEAF1_ECX_AVX, + .leaf7_ebx = LEAF7_EBX_AVX2, + .xcr0 = XCR0_XMM_YMM, +}; + +constexpr X86Features MLKEM_X86_NATIVE{ + .max_basic_leaf = 7, + .leaf1_ecx = LEAF1_ECX_OSXSAVE | LEAF1_ECX_AVX | LEAF1_ECX_SSSE3 | LEAF1_ECX_SSE41 | LEAF1_ECX_POPCNT, + .leaf7_ebx = LEAF7_EBX_AVX2 | LEAF7_EBX_BMI2, + .xcr0 = XCR0_XMM_YMM, +}; + +/** A synthetic CPU for DetectX86Features: answers CPUID like hardware and records every query. */ +struct FakeCpu { + X86Features features; + std::vector leaves; + int xgetbv_calls{0}; + bool xgetbv_faulted{false}; +}; +FakeCpu g_cpu; + +void FakeCpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ + g_cpu.leaves.push_back(leaf); + const bool supported{leaf <= g_cpu.features.max_basic_leaf}; + a = leaf == 0 ? g_cpu.features.max_basic_leaf : 0; + // Real CPUs answer an unsupported leaf with another leaf's data; report + // AVX2 there so that reading it would show up as a wrong result. + b = leaf == 7 ? (supported ? g_cpu.features.leaf7_ebx : LEAF7_EBX_AVX2) : 0; + c = leaf == 1 ? g_cpu.features.leaf1_ecx : 0; + d = 0; + (void)subleaf; +} + +uint64_t FakeXGetBV(uint32_t index) +{ + ++g_cpu.xgetbv_calls; + if ((g_cpu.features.leaf1_ecx & LEAF1_ECX_OSXSAVE) == 0) g_cpu.xgetbv_faulted = true; + return index == 0 ? g_cpu.features.xcr0 : 0; +} + +X86Features DetectFake(const X86Features& features) +{ + g_cpu = FakeCpu{}; + g_cpu.features = features; + return DetectX86Features(FakeCpuid, FakeXGetBV); +} + +} // namespace + +BOOST_AUTO_TEST_SUITE(cpu_features_tests) + +BOOST_AUTO_TEST_CASE(usable_avx2_matrix) +{ + static_assert(HasUsableAVX2(AVX2_USABLE)); + static_assert(!HasUsableAVX2(X86Features{})); + + // Clearing any single requirement makes AVX2 unusable. + X86Features f{AVX2_USABLE}; + f.max_basic_leaf = 6; + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_OSXSAVE; + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_AVX; + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.xcr0 = 0x4; // YMM without XMM state + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.xcr0 = 0x3; // x87 and XMM, but no YMM state + BOOST_CHECK(!HasUsableAVX2(f)); + f = AVX2_USABLE; + f.leaf7_ebx &= ~LEAF7_EBX_AVX2; + BOOST_CHECK(!HasUsableAVX2(f)); + + // Unrelated bits, SHA-NI included, change nothing. + f = AVX2_USABLE; + f.leaf7_ebx |= LEAF7_EBX_SHA; + f.xcr0 |= 0xe1; + f.max_basic_leaf = 0x24; + BOOST_CHECK(HasUsableAVX2(f)); + + // Detection reads exactly what the CPU reports. + BOOST_CHECK(HasUsableAVX2(DetectFake(AVX2_USABLE))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 1); + BOOST_CHECK(!g_cpu.xgetbv_faulted); + BOOST_CHECK(g_cpu.leaves == (std::vector{0, 1, 7})); + + // Without OSXSAVE, XGETBV would fault, so it must not run. + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_OSXSAVE; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 0); + BOOST_CHECK(!g_cpu.xgetbv_faulted); + + // Without AVX, XCR0 is irrelevant and is not read either. + f = AVX2_USABLE; + f.leaf1_ecx &= ~LEAF1_ECX_AVX; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 0); + + // The OS does not save YMM state. + f = AVX2_USABLE; + f.xcr0 = 0x3; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 1); + + // Leaf 7 is never queried when the CPU does not support it, even though + // the fake would report AVX2 there. + f = AVX2_USABLE; + f.max_basic_leaf = 6; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK(g_cpu.leaves == (std::vector{0, 1})); + BOOST_CHECK_EQUAL(DetectFake(f).leaf7_ebx, 0U); + + f = AVX2_USABLE; + f.max_basic_leaf = 0; + BOOST_CHECK(!HasUsableAVX2(DetectFake(f))); + BOOST_CHECK(g_cpu.leaves == (std::vector{0})); + BOOST_CHECK_EQUAL(g_cpu.xgetbv_calls, 0); +} + +BOOST_AUTO_TEST_CASE(mlkem_x86_native_matrix) +{ + static_assert(HasMlkemX86Native(MLKEM_X86_NATIVE)); + static_assert(HasUsableAVX2(MLKEM_X86_NATIVE)); + // AVX2 alone is not enough: the assembly also executes SSSE3, SSE4.1, POPCNT and BMI2. + static_assert(!HasMlkemX86Native(AVX2_USABLE)); + + // Clearing any single requirement, the AVX2 ones included, makes the backend unusable. + const auto without_leaf1{[](uint32_t bit) { X86Features f{MLKEM_X86_NATIVE}; f.leaf1_ecx &= ~bit; return f; }}; + const auto without_leaf7{[](uint32_t bit) { X86Features f{MLKEM_X86_NATIVE}; f.leaf7_ebx &= ~bit; return f; }}; + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_SSSE3))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_SSE41))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_POPCNT))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_OSXSAVE))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf1(LEAF1_ECX_AVX))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf7(LEAF7_EBX_BMI2))); + BOOST_CHECK(!HasMlkemX86Native(without_leaf7(LEAF7_EBX_AVX2))); + X86Features f{MLKEM_X86_NATIVE}; + f.max_basic_leaf = 6; + BOOST_CHECK(!HasMlkemX86Native(f)); + f = MLKEM_X86_NATIVE; + f.xcr0 = 0x3; + BOOST_CHECK(!HasMlkemX86Native(f)); + // Masking BMI2 or POPCNT leaves AVX2 usable, which is exactly why AVX2 alone is not checked. + BOOST_CHECK(HasUsableAVX2(without_leaf7(LEAF7_EBX_BMI2))); + BOOST_CHECK(HasUsableAVX2(without_leaf1(LEAF1_ECX_POPCNT))); + + // Unrelated bits change nothing. + f = MLKEM_X86_NATIVE; + f.leaf7_ebx |= LEAF7_EBX_SHA; + f.leaf1_ecx |= uint32_t{1} << 20; // SSE4.2 + BOOST_CHECK(HasMlkemX86Native(f)); + + // Detection gathers every bit the check reads. + BOOST_CHECK(HasMlkemX86Native(DetectFake(MLKEM_X86_NATIVE))); + BOOST_CHECK(!HasMlkemX86Native(DetectFake(without_leaf7(LEAF7_EBX_BMI2)))); + BOOST_CHECK(!HasMlkemX86Native(DetectFake(without_leaf1(LEAF1_ECX_POPCNT)))); +} + +BOOST_AUTO_TEST_CASE(usable_avx2_host) +{ + const X86Features detected{DetectX86Features()}; + BOOST_CHECK_EQUAL(HasUsableAVX2(), HasUsableAVX2(detected)); + // The cached answer is stable. + BOOST_CHECK_EQUAL(HasUsableAVX2(), HasUsableAVX2()); +#if defined(HAVE_GETCPUID) && (defined(__GNUC__) || defined(__clang__)) + BOOST_TEST_MESSAGE("host: avx2=" << HasUsableAVX2() << " mlkem_x86_native=" << HasMlkemX86Native() + << " sha_ni=" << ((detected.leaf7_ebx & LEAF7_EBX_SHA) != 0)); + BOOST_CHECK_EQUAL(HasMlkemX86Native(), HasMlkemX86Native(detected)); +#ifdef HAVE_BUILTIN_CPU_SUPPORTS + // The compiler's own detection, as an independent check. + __builtin_cpu_init(); + BOOST_CHECK_EQUAL(HasUsableAVX2(), __builtin_cpu_supports("avx2") != 0); + BOOST_CHECK_EQUAL(HasMlkemX86Native(), __builtin_cpu_supports("avx2") && __builtin_cpu_supports("ssse3") && + __builtin_cpu_supports("sse4.1") && __builtin_cpu_supports("popcnt") && + __builtin_cpu_supports("bmi2")); +#endif +#else + // Non-x86 targets and unsupported compilers never report AVX2. + BOOST_CHECK(!HasUsableAVX2()); + BOOST_CHECK(!HasMlkemX86Native()); + BOOST_CHECK_EQUAL(detected.max_basic_leaf, 0U); +#endif +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index fdf0081c26..6952990600 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(bitcoin_util STATIC EXCLUDE_FROM_ALL threadnames.cpp time.cpp tokenpipe.cpp + ../compat/cpu_features.cpp ../logging.cpp ../random.cpp ../randomenv.cpp From 3771bd4ec3ac66ef9e64172ad44fe75b6fb4a26e Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 16:06:11 -0400 Subject: [PATCH 03/20] crypto: require SSSE3 for the SSE4 and SHA-NI SHA256 paths SHA256AutoDetect selects the SSE4 code whenever CPUID leaf 1 reports SSE4.1, and only under that check also the 4-way SSE4.1 code and the SHA-NI code. All three execute SSSE3 instructions. An audit of the built objects finds pshufb and palignr in the SSE4 and SHA-NI code and pshufb in the 4-way code, besides SSE4.1's pinsrd, pextrd and pblendw; no SSE4.2 instruction occurs. A virtual machine can report SSE4.1 while masking SSSE3, and the self-test then dies with SIGILL at startup. Require SSSE3 as well. Under qemu-x86_64, -cpu Haswell,-ssse3 and the SHA-NI model EPYC,-ssse3 die with SIGILL before this change and select the standard implementation after it. Unmasked, Haswell still selects sse4, sse41 and avx2, and EPYC and Denverton x86_shani. This is an upstream Bitcoin Core bug, so the change is kept minimal enough to go upstream. Part of #184 (part 3, #190). Co-Authored-By: Claude Opus 5.5 --- src/crypto/sha256.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 9763d8bc10..3fcaa22ea7 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -601,7 +601,12 @@ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implem uint32_t eax, ebx, ecx, edx; GetCPUID(1, 0, eax, ebx, ecx, edx); if (use_implementation & sha256_implementation::USE_SSE4) { - have_sse4 = (ecx >> 19) & 1; + // Fixes an upstream Bitcoin Core bug, which checks SSE4.1 only. The + // SSE4 code and the SSE4.1 and SHA-NI code chosen under this check + // also execute SSSE3 instructions (pshufb, palignr), which a CPU with + // SSE4.1 need not report: a virtual machine can mask SSSE3 alone, and + // the self-test below would then die with SIGILL. + have_sse4 = ((ecx >> 19) & 1) && ((ecx >> 9) & 1); } have_xsave = (ecx >> 27) & 1; have_avx = (ecx >> 28) & 1; From 8b054de90645b9523db980e58369fb22bb6b8b74 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 13:13:41 -0400 Subject: [PATCH 04/20] crypto: add ML-KEM-1024 wrapper and runtime backend dispatch Build the vendored mlkem-native as ML-KEM-1024 into bitcoin_node only, behind a small C++ wrapper. Nothing in qbitd calls it yet. - src/crypto/mlkem_config.h configures the library (MLK_CONFIG_FILE): parameter set 1024, the qbit_mlkem1024 prefix, deterministic APIs only, zeroization through memory_cleanse, and a capability hook. - Native backends are chosen by qbit-owned backend headers instead of upstream's compile-time defaults, which enable x86_64 only for AVX2 compilers. No source is compiled with an ISA flag: every native entry point asks the hook, which reads only cached CPU detection and one atomic portable override. On x86_64 that detection requires every extension the assembly executes (AVX2, SSSE3, SSE4.1, POPCNT, BMI2), not only the AVX2 that upstream's capability name says. - Upstream's AArch64 single-lane Keccak entry point skips the hook, so the override could not reach it. qbit mirrors that header's dispatch glue with the check added; every other entry point checks itself. - mlkem::KeyGen, CheckPublicKey, Encaps and Decaps take fixed-size spans, map upstream's invalid-key results separately from internal failures, and clear their outputs on every failure. GetBackendNames() reports the implementation that runs. ForcePortableForTesting and InjectResultForTesting are test-only guards. - cmake/mlkem-native.cmake adds WITH_MLKEM_NATIVE=AUTO|ON|OFF and the configure summary line. The library gets core_interface's hardening flags but not qbit's C++ warnings, then APPEND_CPPFLAGS and APPEND_CFLAGS, so the MSan jobs' -U_FORTIFY_SOURCE applies to it as to the other C libraries. MSVC builds without assembly, MSan builds portable C, and AArch64 ELF targets build portable C because the upstream assembly has no BTI landing pads or GNU property note and would strip BTI/PAC marking from the binary (doc/subtrees has the evidence and the revisit trigger). ON fails where native is impossible. mlkem_tests covers NIST ACVP keygen, encapsulation, decapsulation and key checks, all 1040 C2SP CCTV modulus vectors and the 3328/3329 boundary, implicit rejection against an independent SHAKE256, injected failures, moves and wiping, and backend selection, each on the native backend and with portable C forced. ci/checks/test_mlkem_build_policy.py checks the WITH_MLKEM_NATIVE matrix, builds the library with each compiler found (core-checks requires the AArch64 cross compiler), and counts assembly calls to show that the portable override reaches every x86_64 routine. Part of #184 (part 3, #190, D3). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/core-checks.yml | 28 ++ CMakeLists.txt | 7 + ci/checks/test_mlkem_build_policy.py | 583 ++++++++++++++++++++++ cmake/bitcoin-build-config.h.in | 4 + cmake/mlkem-native.cmake | 239 +++++++++ contrib/devtools/update-mlkem-native.sh | 8 +- doc/subtrees/mlkem-native.md | 169 +++++++ src/CMakeLists.txt | 3 + src/crypto/mlkem.cpp | 195 ++++++++ src/crypto/mlkem.h | 153 ++++++ src/crypto/mlkem_arith_backend.h | 29 ++ src/crypto/mlkem_backend.c | 29 ++ src/crypto/mlkem_config.h | 89 ++++ src/crypto/mlkem_fips202_backend.h | 55 +++ src/crypto/mlkem_shim.h | 44 ++ src/test/CMakeLists.txt | 2 + src/test/data/mlkem1024_vectors.json | 144 ++++++ src/test/mlkem_tests.cpp | 618 ++++++++++++++++++++++++ 18 files changed, 2396 insertions(+), 3 deletions(-) create mode 100755 ci/checks/test_mlkem_build_policy.py create mode 100644 cmake/mlkem-native.cmake create mode 100644 src/crypto/mlkem.cpp create mode 100644 src/crypto/mlkem.h create mode 100644 src/crypto/mlkem_arith_backend.h create mode 100644 src/crypto/mlkem_backend.c create mode 100644 src/crypto/mlkem_config.h create mode 100644 src/crypto/mlkem_fips202_backend.h create mode 100644 src/crypto/mlkem_shim.h create mode 100644 src/test/data/mlkem1024_vectors.json create mode 100644 src/test/mlkem_tests.cpp diff --git a/.github/workflows/core-checks.yml b/.github/workflows/core-checks.yml index fc69e7a364..9636846961 100644 --- a/.github/workflows/core-checks.yml +++ b/.github/workflows/core-checks.yml @@ -165,6 +165,34 @@ jobs: - name: Test integrated libbitcoinpqc build policy run: python3 ci/checks/test_libbitcoinpqc_build_policy.py + # Every compiler the build-policy test uses, each listed as required so + # that none of its cases can skip: the AArch64 cross compiler proves that + # AArch64 ELF builds refuse native ML-KEM (no BTI landing pads), MinGW and + # Clang cover the Windows and Clang builds, and cc -m32 an x86 target + # without a backend. gcc-aarch64-linux-gnu only recommends its libc + # headers, so they are installed explicitly. For -m32, libc6-dev-i386 + # provides the 32-bit headers: gcc-multilib conflicts with the AArch64 + # cross compiler on Ubuntu 24.04. clang-21 comes from apt.llvm.org, as in + # ci/test/01_base_install.sh. + - name: Test integrated mlkem-native build policy + env: + MLKEM_POLICY_REQUIRED_COMPILERS: cc cc-m32 clang-21 aarch64-linux-gnu-gcc x86_64-w64-mingw32-gcc-posix + APT_LLVM_V: 21 + run: | + sudo install -d -m 0755 /etc/apt/keyrings + curl --fail --silent --show-error --location --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key \ + | sudo tee /etc/apt/keyrings/apt.llvm.org.asc > /dev/null + codename="$(. /etc/os-release && echo "${VERSION_CODENAME}")" + echo "deb [signed-by=/etc/apt/keyrings/apt.llvm.org.asc] http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${APT_LLVM_V} main" \ + | sudo tee "/etc/apt/sources.list.d/llvm-toolchain-${codename}-${APT_LLVM_V}.list" > /dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + gcc-aarch64-linux-gnu libc6-dev-arm64-cross \ + gcc-mingw-w64-x86-64-posix \ + libc6-dev-i386 \ + "clang-${APT_LLVM_V}" "llvm-${APT_LLVM_V}" + python3 ci/checks/test_mlkem_build_policy.py -v + - name: Test IBD timeout wiring run: python3 ci/checks/test_ibd_timeout_wiring.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 55e4117434..720f071bbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,9 @@ if(WITH_USDT) find_package(USDT MODULE REQUIRED) endif() +set(WITH_MLKEM_NATIVE "AUTO" CACHE STRING "Build the x86_64 and AArch64 ML-KEM assembly, selected at run time: AUTO, ON or OFF.") +set_property(CACHE WITH_MLKEM_NATIVE PROPERTY STRINGS AUTO ON OFF) + option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." ON) cmake_dependent_option(WITH_QRENCODE "Enable QR code support." ON "BUILD_GUI" OFF) @@ -426,6 +429,9 @@ include(cmake/introspection.cmake) include(cmake/ccache.cmake) +include(cmake/mlkem-native.cmake) +mlkem_native_configure() + add_library(warn_interface INTERFACE) target_link_libraries(core_interface INTERFACE warn_interface) if(MSVC) @@ -689,6 +695,7 @@ message(" IPC ................................. ${ipc_status}") message(" USDT tracing ........................ ${WITH_USDT}") message(" QR code (GUI) ....................... ${WITH_QRENCODE}") message(" DBus (GUI) .......................... ${WITH_DBUS}") +message(" ML-KEM-1024 backend ................. ${MLKEM_NATIVE_SUMMARY}") message("Tests:") message(" test_qbit ........................... ${BUILD_TESTS}") message(" test_qbit-qt ........................ ${BUILD_GUI_TESTS}") diff --git a/ci/checks/test_mlkem_build_policy.py b/ci/checks/test_mlkem_build_policy.py new file mode 100755 index 0000000000..7806d74ab7 --- /dev/null +++ b/ci/checks/test_mlkem_build_policy.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Regression tests for qbit's mlkem-native build policy (cmake/mlkem-native.cmake). + +Three layers: +- the WITH_MLKEM_NATIVE decision across the architecture, toolchain and + sanitizer matrix, run in CMake script mode, so every target is covered on + any host; +- configure and build of the mlkem_native library with each compiler found on + this host (cross compilers included), checking which sources and flags are + used; +- the guards in src/crypto/mlkem_config.h that stop a build from compiling + native code where the policy forbids it. + +Cases that need a missing compiler are skipped, unless the compiler is listed +in MLKEM_POLICY_REQUIRED_COMPILERS (space-separated), as CI does for the ones +it installs: then a missing one fails. "cc-m32" stands for cc -m32 with 32-bit +headers. Failures print the signature, cause and fix. + +A fourth layer runs on x86_64 hosts: it links qbit's glue and wrapper with +every assembly routine wrapped by a call counter, and checks that the portable +override reaches each of them. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MODULE = REPO_ROOT / "cmake" / "mlkem-native.cmake" +REQUIRED_COMPILERS = frozenset(os.environ.get("MLKEM_POLICY_REQUIRED_COMPILERS", "").split()) + + +def skip_missing_compiler(test: unittest.TestCase, executable: str, problem: str = "not found") -> None: + if executable in REQUIRED_COMPILERS: + test.fail(f"FAIL: {executable} {problem}\nCause: MLKEM_POLICY_REQUIRED_COMPILERS lists it\n" + f"Fix: install it, or drop it from MLKEM_POLICY_REQUIRED_COMPILERS") + test.skipTest(f"{executable} {problem}") + +SUPPORTED_ARCHS = ("x86_64", "aarch64-macho") +UNSUPPORTED_ARCHS = ( + "aarch64-elf", + "this target (riscv64)", + "this target (ppc64)", + "this target (powerpc64le)", + "this target (armv7)", + "this target (s390x)", + "this target (i686)", +) +SANITIZER_SETTINGS = ("", "address,undefined", "thread", "memory", "fuzzer,memory", "memory,undefined") + +POLICY_SCRIPT = r""" +include("${MODULE}") +mlkem_native_policy("${MODE}" "${ARCH}" "${MSVC}" "${SANITIZERS}") +message(NOTICE "RESULT enabled=${MLKEM_NATIVE_ENABLED} no_asm=${MLKEM_NATIVE_NO_ASM} reason=${MLKEM_NATIVE_REASON}") +""" + +FIXTURE_CMAKELISTS = r""" +cmake_minimum_required(VERSION 3.22) +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) +project(qbit_mlkem_policy LANGUAGES NONE) + +if(NOT DEFINED QBIT_SOURCE_DIR) + message(FATAL_ERROR "QBIT_SOURCE_DIR is required") +endif() +set(PROJECT_SOURCE_DIR "${QBIT_SOURCE_DIR}") +set(WITH_MLKEM_NATIVE "AUTO" CACHE STRING "") + +# Stand-ins for qbit's interface libraries: the hardening marker must reach +# mlkem_native, the C++-only warning must not. +add_library(core_interface INTERFACE) +# Like core_interface's -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3, which the MSan +# jobs undo with APPEND_CPPFLAGS=-U_FORTIFY_SOURCE. +target_compile_options(core_interface INTERFACE -DQBIT_POLICY_CORE_FLAG -UQBIT_POLICY_FORTIFY -DQBIT_POLICY_FORTIFY=3) +add_library(warn_interface INTERFACE) +target_compile_options(warn_interface INTERFACE -Wsuggest-override) +target_link_libraries(core_interface INTERFACE warn_interface) +add_library(sanitize_interface INTERFACE) + +include("${QBIT_SOURCE_DIR}/cmake/mlkem-native.cmake") +mlkem_native_configure() +# Like CMakeLists.txt on MinGW, where the compiler accepts it (GCC's assembler +# does, Clang's integrated one does not). +if(MINGW AND CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(core_interface INTERFACE -Wa,-muse-unaligned-vector-move) + message(NOTICE "CORE_UNALIGNED_VECTOR_MOVE") +endif() +message(NOTICE "SUMMARY ${MLKEM_NATIVE_SUMMARY}") +message(NOTICE "ENABLE_MLKEM_NATIVE=${ENABLE_MLKEM_NATIVE}") +add_mlkem_native() +""" + +# Compiler setups for the configure-and-build layer: (name, executable, +# extra CMake definitions, target architecture, whether AUTO builds native code). +# A setup without extra definitions targets this host. AArch64 ELF is portable +# only (no BTI landing pads in the assembly); macOS arm64, which builds native +# code, needs an SDK and is covered by the CI job on Apple hardware. +COMPILERS = ( + ("host cc", "cc", (), "x86_64", True), + ("clang-21", "clang-21", (), "x86_64", True), + ("aarch64 gcc", "aarch64-linux-gnu-gcc", ("-DCMAKE_SYSTEM_NAME=Linux", "-DCMAKE_SYSTEM_PROCESSOR=aarch64"), "aarch64", False), + ("clang-21 --target=aarch64-linux-gnu", "clang-21", + ("-DCMAKE_SYSTEM_NAME=Linux", "-DCMAKE_SYSTEM_PROCESSOR=aarch64", "-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu"), "aarch64", False), + ("mingw-w64 gcc", "x86_64-w64-mingw32-gcc-posix", ("-DCMAKE_SYSTEM_NAME=Windows", "-DCMAKE_SYSTEM_PROCESSOR=x86_64"), "x86_64", True), + ("clang-21 --target=x86_64-w64-mingw32", "clang-21", + ("-DCMAKE_SYSTEM_NAME=Windows", "-DCMAKE_SYSTEM_PROCESSOR=x86_64", "-DCMAKE_C_COMPILER_TARGET=x86_64-w64-mingw32"), "x86_64", True), +) + + +def run(command: list[str], **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=False, capture_output=True, text=True, **kwargs) + + +def host_arch() -> str: + machine = run(["uname", "-m"]).stdout.strip() + return {"amd64": "x86_64", "arm64": "aarch64"}.get(machine, machine) + + +class MlkemPolicyMatrixTest(unittest.TestCase): + """WITH_MLKEM_NATIVE decisions, in CMake script mode.""" + + def policy(self, mode: str, arch: str, msvc: bool, sanitizers: str) -> subprocess.CompletedProcess[str]: + cmake = shutil.which("cmake") + self.assertIsNotNone(cmake, "cmake is required") + assert cmake is not None + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "policy.cmake" + script.write_text(POLICY_SCRIPT, encoding="utf8") + return run([ + cmake, + f"-DMODULE={MODULE}", + f"-DMODE={mode}", + f"-DARCH={arch}", + f"-DMSVC={'TRUE' if msvc else 'FALSE'}", + f"-DSANITIZERS={sanitizers}", + "-P", str(script), + ]) + + def expect(self, mode: str, arch: str, msvc: bool, sanitizers: str) -> tuple[bool, bool, str | None]: + """The policy as documented in cmake/mlkem-native.cmake: (enabled, no_asm, configure error).""" + no_asm = msvc + blocked = msvc or arch not in SUPPORTED_ARCHS or re.search(r"(^|,)memory($|,)", sanitizers) is not None + if mode.upper() == "OFF": + return False, no_asm, None + if blocked: + if mode.upper() == "ON": + return False, no_asm, "WITH_MLKEM_NATIVE=ON, but" + return False, no_asm, None + return True, no_asm, None + + def test_matrix(self) -> None: + for mode in ("AUTO", "ON", "OFF", "auto"): + for arch in SUPPORTED_ARCHS + UNSUPPORTED_ARCHS: + for msvc in (False, True): + for sanitizers in SANITIZER_SETTINGS: + with self.subTest(mode=mode, arch=arch, msvc=msvc, sanitizers=sanitizers): + result = self.policy(mode, arch, msvc, sanitizers) + combined = result.stdout + result.stderr + enabled, no_asm, error = self.expect(mode, arch, msvc, sanitizers) + if error: + self.assertNotEqual(result.returncode, 0, + f"FAIL: WITH_MLKEM_NATIVE={mode} configured for {arch}, msvc={msvc}, " + f"SANITIZERS={sanitizers!r}\nCause: ON must refuse targets without native code\n" + f"Fix: mlkem_native_policy in {MODULE}\n{combined}") + self.assertIn(error, combined) + continue + self.assertEqual(result.returncode, 0, combined) + match = re.search(r"RESULT enabled=(\w+) no_asm=(\w+) reason=(.*)", combined) + self.assertIsNotNone(match, combined) + assert match is not None + self.assertEqual(match.group(1) == "TRUE", enabled, + f"FAIL: native={match.group(1)} for {mode}/{arch}/msvc={msvc}/{sanitizers!r}\n" + f"Cause: policy disagrees with the documented matrix\nFix: mlkem_native_policy in {MODULE}") + self.assertEqual(match.group(2) == "TRUE", no_asm, combined) + self.assertTrue(match.group(3).strip(), "every decision states its reason") + + def test_invalid_mode_fails(self) -> None: + for mode in ("YES", "1", "native", ""): + with self.subTest(mode=mode): + result = self.policy(mode, "x86_64", False, "") + self.assertNotEqual(result.returncode, 0) + self.assertIn("WITH_MLKEM_NATIVE must be AUTO, ON or OFF", result.stdout + result.stderr) + + +class MlkemBuildTest(unittest.TestCase): + """Configure and build mlkem_native with the compilers on this host.""" + + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.tmp_path = Path(self.tmp.name) + self.source = self.tmp_path / "source" + self.source.mkdir() + (self.source / "CMakeLists.txt").write_text(FIXTURE_CMAKELISTS, encoding="utf8") + self.cmake = shutil.which("cmake") + self.assertIsNotNone(self.cmake, "cmake is required") + self.assertIsNotNone(shutil.which("ninja"), "ninja is required") + + def configure(self, name: str, compiler: str, *definitions: str, + env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + assert self.cmake is not None + return run([ + self.cmake, "-S", str(self.source), "-B", str(self.tmp_path / name), "-G", "Ninja", + f"-DQBIT_SOURCE_DIR={REPO_ROOT}", f"-DCMAKE_C_COMPILER={compiler}", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + *definitions, + ], env=env) + + def compile_flags(self, name: str) -> dict[str, list[str]]: + """The flags of mlkem_native's compile commands by source name, without the file arguments.""" + commands = run(["ninja", "-C", str(self.tmp_path / name), "-t", "commands", "mlkem_native"]) + self.assertEqual(commands.returncode, 0, commands.stdout + commands.stderr) + flags: dict[str, list[str]] = {} + for line in commands.stdout.splitlines(): + words = line.split() + if "-c" not in words: + continue + kept = [] + skip = False + for word in words[1:]: + if skip: + skip = False + elif word in ("-MT", "-MF", "-o", "-c"): + skip = True + elif word != "-MD": + kept.append(word) + flags[Path(words[-1]).name] = kept + return flags + + def build(self, name: str) -> subprocess.CompletedProcess[str]: + assert self.cmake is not None + return run([self.cmake, "--build", str(self.tmp_path / name), "--target", "mlkem_native", "--verbose"]) + + def check_build(self, label: str, compiler: str, extra: tuple[str, ...], mode: str, expect_native: bool) -> str: + name = re.sub(r"\W+", "_", f"{label}_{mode}") + configured = self.configure(name, compiler, f"-DWITH_MLKEM_NATIVE={mode}", + "-DAPPEND_CPPFLAGS=-UQBIT_POLICY_FORTIFY", "-DAPPEND_CFLAGS=-DQBIT_POLICY_APPEND_CFLAG", *extra) + self.assertEqual(configured.returncode, 0, configured.stdout + configured.stderr) + self.assertIn("SUMMARY native" if expect_native else "SUMMARY portable", configured.stdout + configured.stderr) + self.assertIn(f"ENABLE_MLKEM_NATIVE={'TRUE' if expect_native else 'FALSE'}", configured.stdout + configured.stderr) + unaligned_moves = "CORE_UNALIGNED_VECTOR_MOVE" in configured.stdout + configured.stderr + if "-DCMAKE_SYSTEM_NAME=Windows" in extra and Path(compiler).name.endswith(("gcc", "gcc-posix")): + self.assertTrue(unaligned_moves, "the fixture must pass -Wa,-muse-unaligned-vector-move for MinGW GCC, as CMakeLists.txt does") + built = self.build(name) + log = built.stdout + built.stderr + self.assertEqual(built.returncode, 0, + f"FAIL: mlkem_native does not build with {label}, WITH_MLKEM_NATIVE={mode}\n{log}") + compile_lines = [line for line in log.splitlines() if " -c " in line] + self.assertTrue(compile_lines, log) + for line in compile_lines: + self.assertNotRegex(line, r"(^|\s)-m(avx|avx2|arch=|cpu=)", + f"FAIL: an ISA flag reached an mlkem_native source with {label}\n" + f"Cause: native code must be selected at run time\nFix: remove the flag from {MODULE}\n{line}") + self.assertIn("-DQBIT_POLICY_CORE_FLAG", line, "core_interface's own options must apply to C and assembly") + self.assertNotIn("-Wsuggest-override", line, "qbit's C++ warning set must not reach the vendored C") + # APPEND_CPPFLAGS and APPEND_CFLAGS come after core_interface's options, so they can undo one + # (the MSan jobs pass -U_FORTIFY_SOURCE against core_interface's -D_FORTIFY_SOURCE=3). + words = line.split() + last_undefine = len(words) - 1 - words[::-1].index("-UQBIT_POLICY_FORTIFY") + self.assertGreater(last_undefine, words.index("-DQBIT_POLICY_FORTIFY=3"), + f"FAIL: APPEND_CPPFLAGS did not reach an mlkem_native compile after core_interface's options with {label}\n" + f"Cause: the C and assembly sources do not get the APPEND flags, or CMake de-duplicated them\n" + f"Fix: add_mlkem_native in {MODULE}\n{line}") + self.assertIn("-DQBIT_POLICY_APPEND_CFLAG", words, line) + asm_lines = [line for line in compile_lines if "mlkem_native_asm.S" in line] + if unaligned_moves: + # The flag would re-encode the proved assembly; the C sources keep it. + for line in compile_lines: + self.assertEqual("-Wa,-muse-unaligned-vector-move" in line.split(), line not in asm_lines, + f"FAIL: -Wa,-muse-unaligned-vector-move must reach mlkem_native's C sources and not its assembly ({label})\n" + f"Cause: gas would encode the assembly's vmovdqa as vmovdqu, which the HOL-Light proofs do not cover\n" + f"Fix: add_mlkem_native in {MODULE}\n{line}") + if expect_native: + self.assertEqual(len(asm_lines), 1, log) + for line in compile_lines: + self.assertIn("-DQBIT_MLKEM_NATIVE", line, line) + else: + self.assertEqual(asm_lines, [], log) + for line in compile_lines: + self.assertNotIn("-DQBIT_MLKEM_NATIVE", line, line) + return log + + def test_compilers(self) -> None: + found = 0 + for label, executable, extra, arch, native in COMPILERS: + compiler = shutil.which(executable) + with self.subTest(compiler=label): + if compiler is None: + skip_missing_compiler(self, executable) + if not extra and arch != host_arch(): + self.skipTest(f"{label} targets {arch}, not this host") + assert compiler is not None + found += 1 + self.check_build(label, compiler, extra, "OFF", False) + log = self.check_build(label, compiler, extra, "AUTO", native) + target = next((d.split("=", 1)[1] for d in extra if d.startswith("-DCMAKE_C_COMPILER_TARGET=")), None) + if native: + self.check_build(label, compiler, extra, "ON", True) + if target: + asm_line = next(line for line in log.splitlines() if "mlkem_native_asm.S" in line) + self.assertRegex(asm_line, rf"(--target=|-target ){re.escape(target)}\b", + "FAIL: assembly built without the C compiler's target\n" + "Cause: CMAKE_ASM_COMPILER_TARGET not propagated\nFix: mlkem_native_configure") + else: + on = self.configure(re.sub(r"\W+", "_", f"{label}_ON"), compiler, "-DWITH_MLKEM_NATIVE=ON", *extra) + combined = re.sub(r"\s+", " ", on.stdout + on.stderr) + self.assertNotEqual(on.returncode, 0, combined) + self.assertIn("WITH_MLKEM_NATIVE=ON, but", combined) + if arch == "aarch64": + self.assertIn("no BTI landing pads", combined) + self.assertGreater(found, 0, "no usable C compiler found") + + def test_memory_sanitizer(self) -> None: + compiler = shutil.which("cc") + self.assertIsNotNone(compiler) + assert compiler is not None + auto = self.configure("msan_auto", compiler, "-DSANITIZERS=memory") + self.assertEqual(auto.returncode, 0, auto.stdout + auto.stderr) + self.assertIn("ENABLE_MLKEM_NATIVE=FALSE", auto.stdout + auto.stderr) + self.assertIn("MemorySanitizer", auto.stdout + auto.stderr) + built = self.build("msan_auto") + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + self.assertNotIn("mlkem_native_asm.S", built.stdout + built.stderr) + + on = self.configure("msan_on", compiler, "-DSANITIZERS=fuzzer,memory", "-DWITH_MLKEM_NATIVE=ON") + self.assertNotEqual(on.returncode, 0, on.stdout + on.stderr) + self.assertIn("WITH_MLKEM_NATIVE=ON, but MemorySanitizer", re.sub(r"\s+", " ", on.stdout + on.stderr)) + + def test_asm_flags_keep_explicit_configuration(self) -> None: + """The assembly gets the C flags, followed by the ASM flags the user, the + environment or the toolchain set; per configuration, only CMake's own + default gives way to the C flags.""" + if host_arch() != "x86_64": + self.skipTest("the host compiler builds no native ML-KEM code on this host") + compiler = shutil.which("cc") + self.assertIsNotNone(compiler) + assert compiler is not None + toolchain = self.tmp_path / "asm-toolchain.cmake" + toolchain.write_text('set(CMAKE_ASM_FLAGS_INIT "-DTOOLCHAIN_ASM_FLAG")\n' + 'set(CMAKE_ASM_FLAGS_RELWITHDEBINFO_INIT "-DTOOLCHAIN_ASM_RELWITHDEBINFO")\n', encoding="utf8") + c_flags = ("-DCMAKE_C_FLAGS=-DQBIT_POLICY_C_FLAG", "-DCMAKE_C_FLAGS_RELWITHDEBINFO=-O2 -g -DQBIT_POLICY_C_RELWITHDEBINFO") + clean_env = {key: value for key, value in os.environ.items() if key != "ASMFLAGS"} + # (name, definitions, ASMFLAGS, flags the assembly gets after the C flags, RelWithDebInfo flag) + cases = ( + ("default", (), None, (), "-DQBIT_POLICY_C_RELWITHDEBINFO"), + ("user", ("-DCMAKE_ASM_FLAGS=-DUSER_ASM_FLAG",), None, ("-DUSER_ASM_FLAG",), "-DQBIT_POLICY_C_RELWITHDEBINFO"), + ("environment", (), "-DENV_ASM_FLAG", ("-DENV_ASM_FLAG",), "-DQBIT_POLICY_C_RELWITHDEBINFO"), + ("toolchain", (f"-DCMAKE_TOOLCHAIN_FILE={toolchain}",), None, ("-DTOOLCHAIN_ASM_FLAG",), "-DTOOLCHAIN_ASM_RELWITHDEBINFO"), + ("user per-config", ("-DCMAKE_ASM_FLAGS_RELWITHDEBINFO=-DUSER_ASM_RELWITHDEBINFO",), None, (), "-DUSER_ASM_RELWITHDEBINFO"), + ) + for label, definitions, asmflags, asm_only, config_flag in cases: + with self.subTest(case=label): + name = re.sub(r"\W+", "_", f"asm_flags_{label}") + env = dict(clean_env, ASMFLAGS=asmflags) if asmflags else clean_env + configured = self.configure(name, compiler, *c_flags, *definitions, env=env) + self.assertEqual(configured.returncode, 0, configured.stdout + configured.stderr) + flags = self.compile_flags(name) + asm = flags["mlkem_native_asm.S"] + for flag in ("-DQBIT_POLICY_C_FLAG", *asm_only, config_flag): + self.assertEqual(asm.count(flag), 1, + f"FAIL: {flag} reaches mlkem_native_asm.S {asm.count(flag)} times ({label})\n" + f"Cause: mlkem_native_configure replaced or repeated the ASM flags\nFix: {MODULE}\n{asm}") + for flag in asm_only: + self.assertGreater(asm.index(flag), asm.index("-DQBIT_POLICY_C_FLAG"), + f"FAIL: {flag} comes before the C flags ({label})\n" + f"Cause: explicit ASM flags must win on conflict\nFix: {MODULE}\n{asm}") + if config_flag != "-DQBIT_POLICY_C_RELWITHDEBINFO": + self.assertNotIn("-DQBIT_POLICY_C_RELWITHDEBINFO", asm, + f"FAIL: the C RelWithDebInfo flags replaced an explicit ASM value ({label})\nFix: {MODULE}\n{asm}") + else: + # Apart from explicit ASM flags, the assembly compiles exactly like the C. + self.assertEqual([word for word in asm if word not in asm_only], flags["mlkem_native.c"]) + # A reconfigure, with or without the environment, must not accumulate flags. + for again in (env, clean_env): + reconfigured = self.configure(name, compiler, *c_flags, *definitions, env=again) + self.assertEqual(reconfigured.returncode, 0, reconfigured.stdout + reconfigured.stderr) + self.assertEqual(self.compile_flags(name), flags, + f"FAIL: a reconfigure changed the mlkem_native compile commands ({label})\n" + f"Cause: mlkem_native_configure wrote the ASM flags back to the cache\nFix: {MODULE}") + + def test_asm_flags_with_depends_toolchain(self) -> None: + """A toolchain rendered from depends/toolchain.cmake.in, as depends/Makefile + does, assembles with exactly the flags of the C sources.""" + # (name, C compiler, tool prefix, cross: (system name, arch, host), whether AUTO builds native code) + setups = ( + ("native", "cc", "", None, True), + ("mingw-w64", "x86_64-w64-mingw32-gcc-posix", "x86_64-w64-mingw32-", ("Windows", "x86_64", "x86_64-w64-mingw32"), True), + ("aarch64 linux", "aarch64-linux-gnu-gcc", "aarch64-linux-gnu-", ("Linux", "aarch64", "aarch64-linux-gnu"), False), + ) + template = (REPO_ROOT / "depends" / "toolchain.cmake.in").read_text(encoding="utf8") + clean_env = {key: value for key, value in os.environ.items() if key not in ("ASMFLAGS", "CFLAGS")} + for label, executable, prefix, cross, native in setups: + with self.subTest(toolchain=label): + compiler = shutil.which(executable) + if compiler is None: + skip_missing_compiler(self, executable) + assert compiler is not None + if cross is None and host_arch() != "x86_64": + self.skipTest("the host compiler builds no native ML-KEM code on this host") + system, arch, host = cross or ("", "", "") + values = { + "depends_crosscompiling": "TRUE" if cross else "FALSE", "host": host, + "host_system_name": system, "host_system_version": "", "host_arch": arch, + "CC": compiler, "CXX": compiler, "OSX_SDK": "", + "CFLAGS": "-pipe -DQBIT_DEPENDS_CFLAG", "CFLAGS_RELEASE": "-O2 -DQBIT_DEPENDS_RELEASE_CFLAG", + "CFLAGS_DEBUG": "-O1", "CXXFLAGS": "", "CXXFLAGS_RELEASE": "", "CXXFLAGS_DEBUG": "", + "CPPFLAGS": "", "CPPFLAGS_RELEASE": "", "CPPFLAGS_DEBUG": "", + "LDFLAGS": "", "LDFLAGS_RELEASE": "", "LDFLAGS_DEBUG": "", + **{tool.upper(): shutil.which(prefix + tool) or prefix + tool + for tool in ("ar", "ranlib", "strip", "objcopy", "objdump")}, + } + rendered = re.sub(r"@(\w+)@", lambda match: values.get(match.group(1), ""), template) + name = re.sub(r"\W+", "_", f"depends_{label}") + toolchain_dir = self.tmp_path / f"{name}_prefix" + toolchain_dir.mkdir() + (toolchain_dir / "toolchain.cmake").write_text(rendered, encoding="utf8") + configured = self.configure(name, compiler, f"-DCMAKE_TOOLCHAIN_FILE={toolchain_dir / 'toolchain.cmake'}", + env=clean_env) + self.assertEqual(configured.returncode, 0, configured.stdout + configured.stderr) + self.assertIn(f"ENABLE_MLKEM_NATIVE={'TRUE' if native else 'FALSE'}", configured.stdout + configured.stderr) + flags = self.compile_flags(name) + for flag in ("-DQBIT_DEPENDS_CFLAG", "-DQBIT_DEPENDS_RELEASE_CFLAG"): + self.assertIn(flag, flags["mlkem_native.c"], flags) + if not native: + self.assertNotIn("mlkem_native_asm.S", flags) + continue + # Except the MinGW flag that would re-encode the proved assembly (check_build). + c_flags = [flag for flag in flags["mlkem_native.c"] if flag != "-Wa,-muse-unaligned-vector-move"] + self.assertEqual(flags["mlkem_native_asm.S"], c_flags, + f"FAIL: with the {label} depends toolchain the assembly flags differ from the C flags\n" + f"Cause: mlkem_native_configure changed what depends builds pass to the assembler\nFix: {MODULE}") + + +class MlkemConfigHeaderTest(unittest.TestCase): + """src/crypto/mlkem_config.h refuses native code the policy forbids.""" + + def compile_backend(self, compiler: str, *flags: str) -> subprocess.CompletedProcess[str]: + return run([ + compiler, "-fsyntax-only", '-DMLK_CONFIG_FILE="crypto/mlkem_config.h"', + f"-I{REPO_ROOT / 'src'}", f"-I{REPO_ROOT / 'src' / 'mlkem-native'}", + *flags, str(REPO_ROOT / "src" / "crypto" / "mlkem_backend.c"), + ]) + + def test_guards(self) -> None: + cc = shutil.which("cc") + self.assertIsNotNone(cc) + assert cc is not None + self.assertEqual(self.compile_backend(cc).returncode, 0) + both = self.compile_backend(cc, "-DQBIT_MLKEM_NATIVE", "-DMLK_CONFIG_NO_ASM") + self.assertNotEqual(both.returncode, 0) + self.assertIn("QBIT_MLKEM_NATIVE and MLK_CONFIG_NO_ASM", both.stderr) + + cases = ( + ("clang-21", ("-fsanitize=memory", "-DQBIT_MLKEM_NATIVE"), "MemorySanitizer cannot see writes"), + ("aarch64-linux-gnu-gcc", ("-DQBIT_MLKEM_NATIVE",), "no BTI landing pads"), + ("cc", ("-m32", "-DQBIT_MLKEM_NATIVE"), "no native ML-KEM backend"), + ) + for executable, flags, message in cases: + compiler = shutil.which(executable) + with self.subTest(compiler=executable, flags=flags): + if compiler is None: + skip_missing_compiler(self, executable) + assert compiler is not None + if flags[0] == "-m32" and self.compile_backend(compiler, "-m32").returncode != 0: + skip_missing_compiler(self, "cc-m32", "has no 32-bit headers") + result = self.compile_backend(compiler, *flags) + self.assertNotEqual(result.returncode, 0, f"FAIL: {executable} {flags} compiled native code") + self.assertIn(message, result.stderr) + + + +COVERAGE_HARNESS = r""" +#include +#include +#include +unsigned long AsmCalls(); +using namespace mlkem; +static std::vector RoundTrip() +{ + std::array seed{}; + for (size_t i = 0; i < seed.size(); ++i) seed[i] = uint8_t(i * 7 + 1); + const std::array coins{}; + PublicKey ek; DecapsulationKey dk; Ciphertext ct; SharedSecret sent, received; + if (KeyGen(seed, ek, dk) != Error::NONE || CheckPublicKey(ek) != Error::NONE || + Encaps(ek, coins, ct, sent) != Error::NONE || Decaps(dk, ct, received) != Error::NONE) return {}; + std::vector out(ek.begin(), ek.end()); + out.insert(out.end(), ct.begin(), ct.end()); + out.insert(out.end(), received.Bytes().begin(), received.Bytes().end()); + return out; +} +int main() +{ + const auto detected{RoundTrip()}; + const unsigned long detected_calls{AsmCalls()}; + const auto active{GetBackendNames().arith}; + std::vector portable; + unsigned long portable_calls; + { + ForcePortableForTesting force; + portable = RoundTrip(); + portable_calls = AsmCalls(); + } + std::printf("active=%.*s detected_calls=%lu portable_calls=%lu equal=%d ok=%d\n", int(active.size()), active.data(), + detected_calls, portable_calls, int(detected == portable), int(!detected.empty())); +} +""" + + +class MlkemOverrideCoverageTest(unittest.TestCase): + """The portable override reaches every assembly routine of the x86_64 backend. + + Every global symbol the assembly object defines is wrapped (ld --wrap) by a + counter. A missed entry point would still give identical outputs, so only + counting the calls can show it. + """ + + def test_override_reaches_every_assembly_routine(self) -> None: + if host_arch() != "x86_64": + self.skipTest("the x86_64 backend runs only on x86_64 hosts") + cc, cxx, nm = shutil.which("cc"), shutil.which("c++"), shutil.which("nm") + if not (cc and cxx and nm): + self.skipTest("cc, c++ and nm are required") + assert cc is not None and cxx is not None and nm is not None + src = REPO_ROOT / "src" + flags = ['-DMLK_CONFIG_FILE="crypto/mlkem_config.h"', "-DQBIT_MLKEM_NATIVE", f"-I{src}", f"-I{src / 'mlkem-native'}", "-O1"] + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + + def compile_(compiler: str, source: Path, *extra: str) -> Path: + obj = out / (source.name + ".o") + result = run([compiler, *flags, *extra, "-c", str(source), "-o", str(obj)]) + self.assertEqual(result.returncode, 0, result.stderr) + return obj + + objects = [ + compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native.c"), + compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native_asm.S"), + compile_(cc, src / "crypto" / "mlkem_backend.c"), + ] + asm_symbols = sorted(line.split()[2] for line in run([nm, "--defined-only", "-g", str(objects[1])]).stdout.splitlines() + if len(line.split()) == 3 and line.split()[1] == "T") + self.assertGreater(len(asm_symbols), 0, "no assembly routines: is the native backend compiled?") + wrappers = ["#include ", "#include ", "static std::atomic g_calls{0};", 'extern "C" {'] + for symbol in asm_symbols: + # Every routine takes at most six integer or pointer arguments, which the + # SysV ABI passes in registers, so a generic forwarder preserves them. + wrappers.append(f"uint64_t __real_{symbol}(void*, void*, void*, void*, void*, void*);") + wrappers.append(f"uint64_t __wrap_{symbol}(void* a, void* b, void* c, void* d, void* e, void* f) " + f"{{ ++g_calls; return __real_{symbol}(a, b, c, d, e, f); }}") + wrappers += ["}", "unsigned long AsmCalls() { return g_calls.exchange(0); }"] + (out / "wrap.cpp").write_text("\n".join(wrappers) + "\n", encoding="utf8") + (out / "harness.cpp").write_text(COVERAGE_HARNESS, encoding="utf8") + for source in (src / "crypto" / "mlkem.cpp", src / "compat" / "cpu_features.cpp", src / "support" / "cleanse.cpp", + out / "harness.cpp", out / "wrap.cpp"): + objects.append(compile_(cxx, source, "-std=c++20")) + binary = out / "harness" + link = run([cxx, "-o", str(binary), *map(str, objects), *(f"-Wl,--wrap={symbol}" for symbol in asm_symbols)]) + self.assertEqual(link.returncode, 0, link.stderr) + result = run([str(binary)]) + self.assertEqual(result.returncode, 0, result.stderr) + values = dict(item.split("=", 1) for item in result.stdout.split()) + self.assertEqual(values["ok"], "1", result.stdout) + self.assertEqual(values["equal"], "1", "the backends disagree") + self.assertEqual(values["portable_calls"], "0", + f"FAIL: assembly ran {values['portable_calls']} times with portable C forced\n" + "Cause: a native entry point does not ask mlk_sys_check_capability\n" + "Fix: mirror its dispatch glue with a capability check, as for AArch64's x1 Keccak in src/crypto/mlkem_fips202_backend.h") + if values["active"] == "x86_64-avx2": + self.assertGreater(int(values["detected_calls"]), 0, result.stdout) + else: + self.assertEqual(values["detected_calls"], "0", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/bitcoin-build-config.h.in b/cmake/bitcoin-build-config.h.in index 965cb72bac..eb9ee52b0c 100644 --- a/cmake/bitcoin-build-config.h.in +++ b/cmake/bitcoin-build-config.h.in @@ -39,6 +39,10 @@ /* Define to 1 to enable wallet functions. */ #cmakedefine ENABLE_WALLET 1 +/* Define to 1 if mlkem-native's native ML-KEM backends are compiled in, and to + 0 if not. */ +#cmakedefine01 ENABLE_MLKEM_NATIVE + /* Define to 1 if you have the declaration of `fork', and to 0 if you don't. */ #cmakedefine01 HAVE_DECL_FORK diff --git a/cmake/mlkem-native.cmake b/cmake/mlkem-native.cmake new file mode 100644 index 0000000000..299f2b3708 --- /dev/null +++ b/cmake/mlkem-native.cmake @@ -0,0 +1,239 @@ +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +# Builds the vendored mlkem-native library (src/mlkem-native) as ML-KEM-1024. +# See doc/subtrees/mlkem-native.md and src/crypto/mlkem_config.h. +# +# Native backends (x86_64 AVX2 and AArch64 assembly) are selected at run time, +# so no source is ever compiled with an AVX2 flag and the portable C fallback +# runs on every CPU. WITH_MLKEM_NATIVE chooses whether they are built: +# AUTO build them where supported (the default); +# ON like AUTO, but a target that cannot build them is a configure error; +# OFF portable C only. + +include_guard(GLOBAL) + +# Decide whether native backends are built. This depends on its arguments only, +# so ci/checks/test_mlkem_build_policy.py can run it for any target in script +# mode. +# mode WITH_MLKEM_NATIVE: AUTO, ON or OFF +# arch from mlkem_native_detect_arch: x86_64, aarch64-macho, +# aarch64-elf, or a description of another target +# msvc true for MSVC-style toolchains, which cannot assemble GNU syntax +# sanitizers the SANITIZERS value +# +# AArch64 ELF targets (Linux and the BSDs) are portable only: mlkem-native's +# AArch64 assembly has no BTI landing pads and no GNU property note, so linking +# it would strip the BTI and PAC marking from the whole binary. Hardening wins; +# see doc/subtrees/mlkem-native.md for the evidence and when to revisit. +# Sets MLKEM_NATIVE_ENABLED, MLKEM_NATIVE_NO_ASM and MLKEM_NATIVE_REASON in the +# caller's scope. An invalid mode, or ON where native code is impossible, is a +# configure error. +function(mlkem_native_policy mode arch msvc sanitizers) + string(TOUPPER "${mode}" mode) + if(NOT mode MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR "WITH_MLKEM_NATIVE must be AUTO, ON or OFF, not \"${mode}\".") + endif() + + set(no_asm FALSE) + set(blocker "") + if(msvc) + set(no_asm TRUE) + set(blocker "MSVC toolchains cannot assemble mlkem-native's GNU-syntax assembly") + elseif(arch STREQUAL "aarch64-elf") + set(blocker "mlkem-native AArch64 assembly has no BTI landing pads or GNU property note") + elseif(NOT arch STREQUAL "x86_64" AND NOT arch STREQUAL "aarch64-macho") + set(blocker "mlkem-native has no native backend qbit enables for ${arch}") + elseif(sanitizers MATCHES "(^|,)memory($|,)") + set(blocker "MemorySanitizer cannot see writes made by assembly") + endif() + + if(mode STREQUAL "OFF") + set(enabled FALSE) + set(reason "WITH_MLKEM_NATIVE=OFF") + elseif(blocker) + if(mode STREQUAL "ON") + message(FATAL_ERROR + "WITH_MLKEM_NATIVE=ON, but ${blocker}. " + "Configure with WITH_MLKEM_NATIVE=AUTO or OFF to build portable C." + ) + endif() + set(enabled FALSE) + set(reason "${blocker}") + else() + set(enabled TRUE) + set(reason "WITH_MLKEM_NATIVE=${mode} on ${arch}") + endif() + + set(MLKEM_NATIVE_ENABLED ${enabled} PARENT_SCOPE) + set(MLKEM_NATIVE_NO_ASM ${no_asm} PARENT_SCOPE) + set(MLKEM_NATIVE_REASON "${reason}" PARENT_SCOPE) +endfunction() + +# Name the target for mlkem_native_policy by asking the C compiler, with the +# flags in use, which architecture it generates code for. The tests mirror the +# conditions under which src/crypto/mlkem_config.h enables a backend. +function(mlkem_native_detect_arch var) + include(CheckCSourceCompiles) + set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + check_c_source_compiles(" + #if !((defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64)) && !defined(__ILP32__)) + #error not LP64 x86_64 + #endif + int main(void) { return 0; } + " MLKEM_NATIVE_TARGET_X86_64 + ) + check_c_source_compiles(" + #if !((defined(__AARCH64EL__) || defined(_M_ARM64)) && defined(__ARM_NEON) && !defined(__ILP32__)) + #error not little-endian LP64 AArch64 with NEON + #endif + #if !defined(__APPLE__) + #error not Mach-O + #endif + int main(void) { return 0; } + " MLKEM_NATIVE_TARGET_AARCH64_MACHO + ) + check_c_source_compiles(" + #if !((defined(__AARCH64EL__) || defined(_M_ARM64)) && defined(__ELF__)) + #error not AArch64 ELF + #endif + int main(void) { return 0; } + " MLKEM_NATIVE_TARGET_AARCH64_ELF + ) + if(MLKEM_NATIVE_TARGET_X86_64) + set(${var} "x86_64" PARENT_SCOPE) + elseif(MLKEM_NATIVE_TARGET_AARCH64_MACHO) + set(${var} "aarch64-macho" PARENT_SCOPE) + elseif(MLKEM_NATIVE_TARGET_AARCH64_ELF) + set(${var} "aarch64-elf" PARENT_SCOPE) + else() + set(${var} "this target (${CMAKE_SYSTEM_PROCESSOR})" PARENT_SCOPE) + endif() +endfunction() + +# Apply WITH_MLKEM_NATIVE to this build. Call from the top-level directory, at +# file scope, after SANITIZERS is final. Sets MLKEM_NATIVE_ENABLED, +# MLKEM_NATIVE_NO_ASM, MLKEM_NATIVE_REASON and MLKEM_NATIVE_SUMMARY. A macro, +# because enable_language() must run at file scope. +macro(mlkem_native_configure) + enable_language(C) + if(MSVC) + set(_mlkem_native_arch "MSVC") + else() + mlkem_native_detect_arch(_mlkem_native_arch) + endif() + mlkem_native_policy("${WITH_MLKEM_NATIVE}" "${_mlkem_native_arch}" "${MSVC}" "${SANITIZERS}") + if(MLKEM_NATIVE_ENABLED) + # Assemble with exactly the C compiler invocation, including the arguments + # and target that cross toolchains (depends) give it. + set(CMAKE_ASM_COMPILER "${CMAKE_C_COMPILER}") + set(CMAKE_ASM_COMPILER_ARG1 "${CMAKE_C_COMPILER_ARG1}") + if(DEFINED CMAKE_C_COMPILER_TARGET) + set(CMAKE_ASM_COMPILER_TARGET "${CMAKE_C_COMPILER_TARGET}") + endif() + # enable_language(ASM) appends CMake's defaults to a toolchain's + # CMAKE_ASM_FLAGS__INIT, so note now which ones the toolchain set. + foreach(_mlkem_native_config IN ITEMS DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + set(_mlkem_native_init_${_mlkem_native_config} FALSE) + if(DEFINED CMAKE_ASM_FLAGS_${_mlkem_native_config}_INIT) + set(_mlkem_native_init_${_mlkem_native_config} TRUE) + endif() + endforeach() + enable_language(ASM) + # The C flags come first, then the ASM flags CMake initialized from + # -DCMAKE_ASM_FLAGS, the ASMFLAGS environment variable or the toolchain + # (target, sysroot or hardening flags), which win on conflict. Normal + # variables shadow the cache without changing it, so a reconfigure starts + # again from the same values. + string(STRIP "${CMAKE_C_FLAGS} ${CMAKE_ASM_FLAGS}" CMAKE_ASM_FLAGS) + # Per configuration, CMake's own ASM default (its _INIT value) gives way to + # the C flags; a value the user or the toolchain set is kept. + foreach(_mlkem_native_config IN ITEMS DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + string(STRIP "${CMAKE_ASM_FLAGS_${_mlkem_native_config}}" _mlkem_native_value) + string(STRIP "${CMAKE_ASM_FLAGS_${_mlkem_native_config}_INIT}" _mlkem_native_default) + if(NOT _mlkem_native_init_${_mlkem_native_config} AND _mlkem_native_value STREQUAL _mlkem_native_default) + set(CMAKE_ASM_FLAGS_${_mlkem_native_config} "${CMAKE_C_FLAGS_${_mlkem_native_config}}") + endif() + unset(_mlkem_native_init_${_mlkem_native_config}) + endforeach() + unset(_mlkem_native_value) + unset(_mlkem_native_default) + unset(_mlkem_native_config) + if(_mlkem_native_arch STREQUAL "x86_64") + set(MLKEM_NATIVE_SUMMARY "native x86_64-avx2 if the CPU has AVX2, SSSE3, SSE4.1, POPCNT and BMI2, else portable (${MLKEM_NATIVE_REASON})") + else() + set(MLKEM_NATIVE_SUMMARY "native aarch64-neon (${MLKEM_NATIVE_REASON})") + endif() + else() + set(MLKEM_NATIVE_SUMMARY "portable (${MLKEM_NATIVE_REASON})") + endif() + unset(_mlkem_native_arch) + # For bitcoin-build-config.h. + set(ENABLE_MLKEM_NATIVE ${MLKEM_NATIVE_ENABLED}) +endmacro() + +# Create the mlkem_native static library from the vendored tree in +# ${PROJECT_SOURCE_DIR}/src/mlkem-native, as configured by +# mlkem_native_configure(). Link it PRIVATE into bitcoin_node only; its +# interface lets bitcoin_node's ML-KEM wrapper include mlkem/mlkem_native.h. +# +# The library is C and assembly, so it takes core_interface's own compile +# options and definitions (hardening flags such as -fcf-protection and +# -mbranch-protection) and the sanitizer flags, but not qbit's C++ warning +# set: the vendored code is never edited to silence warnings. +function(add_mlkem_native) + set(vendor_dir ${PROJECT_SOURCE_DIR}/src/mlkem-native) + add_library(mlkem_native STATIC EXCLUDE_FROM_ALL + ${vendor_dir}/mlkem/mlkem_native.c + ${PROJECT_SOURCE_DIR}/src/crypto/mlkem_backend.c + ) + if(MLKEM_NATIVE_ENABLED) + target_sources(mlkem_native PRIVATE ${vendor_dir}/mlkem/mlkem_native_asm.S) + target_compile_definitions(mlkem_native PRIVATE QBIT_MLKEM_NATIVE) + endif() + if(MLKEM_NATIVE_NO_ASM) + target_compile_definitions(mlkem_native PRIVATE MLK_CONFIG_NO_ASM) + endif() + + # core_interface's own options and definitions, read directly: a + # $ expression would also pull in warn_interface. + get_target_property(core_options core_interface INTERFACE_COMPILE_OPTIONS) + get_target_property(core_definitions core_interface INTERFACE_COMPILE_DEFINITIONS) + target_compile_definitions(mlkem_native + PUBLIC + "MLK_CONFIG_FILE=\"crypto/mlkem_config.h\"" + ) + if(core_definitions) + target_compile_definitions(mlkem_native PRIVATE ${core_definitions}) + endif() + if(core_options) + # On MinGW, core_interface passes -Wa,-muse-unaligned-vector-move, which + # would make gas encode the assembly's aligned vector moves (vmovdqa) as + # unaligned ones: not the encoding the HOL-Light proofs cover. The assembly + # needs no such protection, because its aligned moves touch only 32-byte + # aligned data, so the flag applies to the C sources only. See + # doc/subtrees/mlkem-native.md. + list(TRANSFORM core_options REPLACE "^(-Wa,-muse-unaligned-vector-move)$" "$<$:\\1>") + target_compile_options(mlkem_native PRIVATE ${core_options}) + endif() + # APPEND_CPPFLAGS and APPEND_CFLAGS reach the C and assembly compiles last, + # after core_interface's options, as for the other C libraries + # (cmake/secp256k1.cmake). The MSan jobs rely on this to undo + # core_interface's -D_FORTIFY_SOURCE with -U_FORTIFY_SOURCE. One SHELL: + # group, because CMake drops an option that repeats an earlier one, and + # core_interface already passes -U_FORTIFY_SOURCE before its -D. + string(STRIP "${APPEND_CPPFLAGS} ${APPEND_CFLAGS}" append_flags) + if(append_flags) + target_compile_options(mlkem_native PRIVATE "SHELL:${append_flags}") + endif() + target_include_directories(mlkem_native + PUBLIC + $ + $ + ) + target_link_libraries(mlkem_native PRIVATE sanitize_interface) + set_target_properties(mlkem_native PROPERTIES + EXPORT_COMPILE_COMMANDS OFF + ) +endfunction() diff --git a/contrib/devtools/update-mlkem-native.sh b/contrib/devtools/update-mlkem-native.sh index 20630919db..10bf8de331 100755 --- a/contrib/devtools/update-mlkem-native.sh +++ b/contrib/devtools/update-mlkem-native.sh @@ -139,9 +139,11 @@ doc/subtrees/mlkem-native.md, in particular: - qbit's src/crypto/mlkem_config.h, mlkem_arith_backend.h and mlkem_fips202_backend.h include upstream paths and config macros; check them against the new upstream tree and its release notes; - - rebuild and run: test_qbit --run_test=mlkem_tests (native and with - MLKEM_FORCE_PORTABLE=1), the mlkem and mlkem_backend_diff fuzz targets, - bench_qbit -filter='MLKEM.*', and ci/checks/test_mlkem_build_policy.py; + - diff upstream's x1_scalar.h against qbit's AArch64 Keccak mirror; + - repeat the x86_64 instruction-set audit of the assembly; + - rebuild and run: test_qbit --run_test=mlkem_tests, the mlkem and + mlkem_backend_diff fuzz targets, bench_qbit -filter='MLKEM.*', and + ci/checks/test_mlkem_build_policy.py; - repeat the readelf GNU property comparison and the Guix build; - after committing, run test/lint/mlkem-native-check.sh. EOF diff --git a/doc/subtrees/mlkem-native.md b/doc/subtrees/mlkem-native.md index 1d18646958..0079a05670 100644 --- a/doc/subtrees/mlkem-native.md +++ b/doc/subtrees/mlkem-native.md @@ -28,6 +28,143 @@ not build, so the identity check stays a single tree id and every file that upstream's single-compilation-unit build (`mlkem_native.c`, `mlkem_native_asm.S`) includes is present. +## How qbit Builds It + +`cmake/mlkem-native.cmake` compiles upstream's single compilation unit +(`mlkem/mlkem_native.c`) and, when native code is enabled, its assembly unit +(`mlkem/mlkem_native_asm.S`) into the `mlkem_native` library, linked privately +into `bitcoin_node` only. Never into `bitcoin_crypto`, `bitcoin_common` or +`bitcoin_consensus`; tests, fuzzers and benchmarks reach it through their node +linkage. All qbit-owned glue lives outside the vendored tree: + +- `src/crypto/mlkem_config.h`: the configuration, selected with + `MLK_CONFIG_FILE`. ML-KEM-1024, the `qbit_mlkem1024` symbol prefix, + deterministic APIs only, zeroization through `memory_cleanse`, and the + capability hook. +- `src/crypto/mlkem_arith_backend.h` and `src/crypto/mlkem_fips202_backend.h`: + the native backends, chosen by qbit rather than by upstream's + compile-time defaults (which enable x86_64 only for AVX2 compilers). +- `src/crypto/mlkem_shim.h` and `src/crypto/mlkem_backend.c`: the C ABI + between the library and qbit's C++. +- `src/crypto/mlkem.{h,cpp}`: the `mlkem::` wrapper. + +Native code is selected at run time. No source is compiled with `-mavx2` or +another ISA flag: every native entry point asks the capability hook first, and +the hook reads only cached CPU detection plus the portable override. The +`WITH_MLKEM_NATIVE` CMake option (`AUTO`, `ON`, `OFF`) decides whether native +code is built at all, and the configure summary's `ML-KEM-1024 backend` line +says what was chosen and why: + +| Target | `AUTO` | `ON` | +|---|---|---| +| x86_64 with GCC or Clang (Linux, macOS, MinGW) | native x86_64 AVX2, used when the CPU has every extension it needs (below) | native | +| macOS arm64 | native AArch64 | native | +| AArch64 ELF (Linux, BSD) | portable (see below) | configure error | +| MSVC | portable, no assembly at all | configure error | +| `SANITIZERS` contains `memory` | portable (MSan cannot see assembly writes) | configure error | +| anything else (riscv64, powerpc64, armv7, s390x, i686, ...) | portable | configure error | + +`ci/checks/test_mlkem_build_policy.py` also links the glue with every x86_64 +assembly routine wrapped by a call counter, and fails if any of them runs +while portable C is forced. + +### x86_64 instruction set + +The x86_64 backend runs only where the CPU has every extension its assembly +executes; `cpu_features::HasMlkemX86Native()` checks them once and caches the +answer, and the capability hook answers upstream's single +`MLK_SYS_CAP_X86_64_AVX2` query with it. Upstream annotates its routines +`Features: [AVX2]`, but an audit of the assembled `mlkem_native_asm.S.o` +(v2.0.0) found: + +| Extension | Instructions | Where | +|---|---|---| +| AVX2 (with AVX, and the OS saving YMM state) | 256-bit integer ops such as `vpmulhw`, `vperm2i128`, `vpblendd`, `vpsrlvq`, `vpbroadcastq`, `vinserti128` | every routine | +| AVX | `vmovdqa`, `vmovsldup`, `vmovhpd`, `vpinsrq`, `vpinsrw`, `vpextrw` | every routine | +| SSSE3 | `pshufb` | `rej_uniform` | +| SSE4.1 | `pinsrd`, `pinsrq`, `pblendw` | `rej_uniform` | +| POPCNT | `popcnt` | `rej_uniform` | +| BMI2 | `pext` | `rej_uniform` | +| baseline x86-64 | SSE2, `cmov`, `rep movsb`; `endbr64` is a no-op without CET | | + +No FMA, AVX-512, BMI1, LZCNT, MOVBE, AES, PCLMUL or SHA instruction occurs. +A virtual machine can show AVX2 while hiding the others; a check of AVX2 alone +would then die with SIGILL on the first ML-KEM operation. qbit's C code is +compiled for baseline x86-64 and uses only SSE2. + +Repeat the audit on every library update, with native code built: + +```bash +objdump -d --no-show-raw-insn build/src/CMakeFiles/mlkem_native.dir/mlkem-native/mlkem/mlkem_native_asm.S.o \ + | awk -F'\t' 'NF >= 2 && $1 ~ /:$/ { split($2, a, " "); print a[1] }' | sort | uniq -c +``` + +Classify every mnemonic, and extend `HasMlkemX86Native` and its tests for a +new extension. Then check masked CPUs: under +`qemu-x86_64 -cpu Haswell,-bmi2` (and `-popcnt`, `-sse4.1`, `-ssse3`) a +program using `mlkem::` must report portable and finish without SIGILL. +`test_qbit` itself runs under all four masks. + +### MinGW assembly encoding + +Windows builds run the x86_64 backend natively too: with GCC or Clang, +upstream declares its SysV-ABI routines `__attribute__((sysv_abi))`, and +`mlkem_native_asm.S` assembles all 15 of them. On MinGW, `core_interface` +passes `-Wa,-muse-unaligned-vector-move` (GCC bug 54412: on 64-bit Windows, +GCC does not keep the stack 32-byte aligned for AVX code). On the assembly, +that flag would make gas encode each of its 632 aligned vector moves +(`vmovdqa`) as an unaligned one (`vmovdqu`): object bytes the HOL-Light proofs +do not cover. `add_mlkem_native` therefore passes the flag to the C sources +only, and `ci/checks/test_mlkem_build_policy.py` checks that it does. + +The assembly needs no such protection, because its aligned moves touch only +32-byte aligned data: + +- `MLK_ALIGN` stack buffers, which MinGW GCC aligns by over-allocating and + rounding the address, since it cannot realign the stack pointer (14 sites in + `mlkem_native.c`, as many as the stack realignments on Linux); +- constant tables, in a 32-byte aligned `.rdata` section. + +No aligned move addresses the stack pointer or RIP-relative data, and the +vendored C is compiled without AVX, so it has no AVX spills. A misaligned +buffer would fault rather than give a wrong result. Recheck this on every +library update, by disassembling both objects built for MinGW. + +### AArch64 ELF builds are portable only + +mlkem-native v2.0.0's AArch64 assembly has no BTI landing pads (`bti c`) and +no GNU property note. A linker keeps an ELF binary's BTI and PAC marking only if +every input object carries it, so linking the native backend would strip the +marking from the whole binary. Hardening wins: AArch64 ELF targets build +portable C. Measured with Ubuntu's `aarch64-linux-gnu-gcc` 15.2.0 and +`-mbranch-protection=standard`, as qbit builds aarch64 Linux: + +```text +mlkem_native.c.o, qbit's C and C++ objects AArch64 feature: BTI, PAC, GCS +mlkem_native_asm.S.o (no GNU property note) +binary with portable ML-KEM AArch64 feature: BTI, PAC, GCS +same binary with the native backend (no AArch64 feature note) +ld -z force-bti: asm.o: warning: BTI is required by -z force-bti, but this + input object file lacks the necessary property note. +``` + +Revisit when an upstream release ships AArch64 assembly with BTI landing pads +and the GNU property note: then re-run the property comparison below on +aarch64 and, if no marking is lost, drop the `aarch64-elf` rule in +`cmake/mlkem-native.cmake` and `src/crypto/mlkem_config.h`. Never restore the +marking by force-marking objects that lack landing pads. macOS arm64 (Mach-O, +no GNU property notes) keeps the native backend. + +### The AArch64 single-lane Keccak mirror + +Upstream's `fips202/native/aarch64/x1_scalar.h` runs its scalar assembly +without asking the capability hook, so qbit's portable override could not reach +it and `mlkem::GetBackendNames()` would report portable Keccak while assembly +ran. `src/crypto/mlkem_fips202_backend.h` therefore mirrors that header's +dispatch glue (the same macros and the same proved assembly routine) with a +capability check added. Every other native entry point in the x86_64 and +AArch64 backends checks the hook itself. + ## Update Procedure Run in a qbit worktree: @@ -73,6 +210,38 @@ confirm: - [ ] `test/lint/mlkem-native-check.sh` prints `GOOD`, and the full lint runner passes. - [ ] Running the update script again at the new tag leaves no diff. +- [ ] The options `src/crypto/mlkem_config.h` sets, the `MLK_SYS_*` macros it + reads and the upstream paths the backend headers include still exist with + the same meaning (upstream `mlkem/mlkem_native_config.h`, `mlkem/src/sys.h`, + `mlkem/src/native/*/meta.h`). +- [ ] Upstream's `mlkem/src/fips202/native/aarch64/x1_scalar.h` was diffed + against the mirror in `src/crypto/mlkem_fips202_backend.h`, and the mirror + still differs only by its capability check. +- [ ] Every native entry point compiled in still asks the capability hook: + `grep -n "static MLK_INLINE\|mlk_sys_check_capability"` over + `mlkem/src/native/x86_64/meta.h`, `mlkem/src/native/aarch64/meta.h`, + `mlkem/src/fips202/native/x86_64/keccak_f1600_x4_avx2.h` and + `mlkem/src/fips202/native/aarch64/x4_v8a_scalar.h` pairs each function with a + check. A new unchecked entry point needs a mirror like the x1 one. +- [ ] The x86_64 instruction-set audit above was repeated, and + `HasMlkemX86Native` still requires every extension the assembly executes. +- [ ] `test_qbit --run_test=mlkem_tests` passes (every case runs on the native + backend and with portable C forced), and so do + `ci/checks/test_mlkem_build_policy.py`, the `mlkem` and `mlkem_backend_diff` + fuzz targets, and `bench_qbit -filter='MLKEM.*'`. +- [ ] The GNU property notes of the release ELF binaries did not lose a + marking. Build the base and the change with the release hardening flags + (Guix for release evidence) and compare, for each binary (`qbitd`, + `qbit-cli`, `qbit-qt`, `test_qbit`, ...): + + ```bash + readelf -n | grep -E 'x86 feature|AArch64 feature' + ``` + + x86_64 must keep `IBT, SHSTK`; aarch64 must keep `BTI, PAC` (and `GCS` where + present). A lost marking blocks the update; see the AArch64 section above. +- [ ] Guix builds of every release host reproduce (two independent builds, + identical hashes). ## Common Failures diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c97d6651be..0e05808cdc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,6 +27,7 @@ include(../cmake/secp256k1.cmake) add_secp256k1(secp256k1) include(../cmake/libbitcoinpqc.cmake) add_libbitcoinpqc(libbitcoinpqc) +add_mlkem_native() # Set top-level target output locations. if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) @@ -193,6 +194,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL blockencodings.cpp blockfilter.cpp consensus/tx_verify.cpp + crypto/mlkem.cpp dbwrapper.cpp deploymentstatus.cpp flatfile.cpp @@ -292,6 +294,7 @@ target_link_libraries(bitcoin_node $ leveldb minisketch + mlkem_native univalue Boost::headers $ diff --git a/src/crypto/mlkem.cpp b/src/crypto/mlkem.cpp new file mode 100644 index 0000000000..c96d693cb6 --- /dev/null +++ b/src/crypto/mlkem.cpp @@ -0,0 +1,195 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include + +#include +#include +#include + +#include + +#include +#include + +static_assert(MLKEM1024_PUBLICKEYBYTES == mlkem::PUBLIC_KEY_BYTES); +static_assert(MLKEM1024_SECRETKEYBYTES == mlkem::SECRET_KEY_BYTES); +static_assert(MLKEM1024_CIPHERTEXTBYTES == mlkem::CIPHERTEXT_BYTES); +static_assert(MLKEM1024_BYTES == mlkem::SHARED_SECRET_BYTES); +static_assert(2 * MLKEM1024_SYMBYTES == mlkem::KEYGEN_SEED_BYTES); +static_assert(MLKEM1024_SYMBYTES == mlkem::ENCAPS_COINS_BYTES); +static_assert(MLK_ERR_FAIL == mlkem::upstream::ERR_FAIL); +static_assert(MLK_ERR_OUT_OF_MEMORY == mlkem::upstream::ERR_OUT_OF_MEMORY); +static_assert(MLK_ERR_RNG_FAIL == mlkem::upstream::ERR_RNG_FAIL); +static_assert(MLK_ERR_INVALID_PK == mlkem::upstream::ERR_INVALID_PK); +static_assert(MLK_ERR_INVALID_SK == mlkem::upstream::ERR_INVALID_SK); +static_assert(MLK_ERR_PCT_FAIL == mlkem::upstream::ERR_PCT_FAIL); + +namespace { + +constexpr std::string_view PORTABLE{"portable"}; + +//! When set, every operation that starts afterwards runs portable C only. +std::atomic_bool g_force_portable{false}; + +/** + * g_force_portable as the operation running on this thread read it at entry. + * The capability hooks read only this copy, so a change on another thread + * mid-operation cannot mix backends: the x86_64 backend keeps its own + * coefficient order, and a mixed operation would give a wrong result. + */ +thread_local bool t_force_portable{false}; + +//! Whether the CPU and OS can run the x86_64 backend. Written once, by DetectCapabilities(). +std::atomic_bool g_cpu_x86_native{false}; + +//! Results injected by InjectResultForTesting, indexed by Operation; 0 means none. +std::array g_injected_results{}; + +/** + * Detect CPU features on the first call. Every entry point calls this before + * entering the library, so no CPUID or XGETBV runs inside an operation, and + * the capability hooks read only the cached result. + */ +void DetectCapabilities() noexcept +{ + static const bool detected{[] { + g_cpu_x86_native.store(cpu_features::HasMlkemX86Native(), std::memory_order_relaxed); + return true; + }()}; + (void)detected; +} + +/** Call at every entry point, before the library runs: fixes the backend for this operation. */ +void BeginOperation() noexcept +{ + DetectCapabilities(); + t_force_portable = g_force_portable.load(std::memory_order_relaxed); +} + +int Injected(mlkem::Operation op, int result) noexcept +{ + const int injected{g_injected_results[static_cast(op)].load(std::memory_order_relaxed)}; + return injected != 0 ? injected : result; +} + +mlkem::Error MapResult(mlkem::Operation op, int result) noexcept +{ + if (result == 0) return mlkem::Error::NONE; + if (result == MLK_ERR_INVALID_PK && (op == mlkem::Operation::CHECK_PUBLIC_KEY || op == mlkem::Operation::ENCAPS)) { + return mlkem::Error::INVALID_PUBLIC_KEY; + } + if (result == MLK_ERR_INVALID_SK && op == mlkem::Operation::DECAPS) return mlkem::Error::INVALID_SECRET_KEY; + // Anything else, including codes this configuration never produces, is an internal failure. + return mlkem::Error::INTERNAL; +} + +/** The backend that runs for a compiled-in backend name, given the capability hooks' current answers. */ +std::string_view ActiveBackend(std::string_view compiled) noexcept +{ + if (compiled == "x86_64-avx2") return qbit_mlkem_has_avx2() ? compiled : PORTABLE; + if (compiled == "aarch64-neon" || compiled == "aarch64") return qbit_mlkem_has_neon() ? compiled : PORTABLE; + return PORTABLE; +} + +} // namespace + +extern "C" void qbit_mlkem_zeroize(void* ptr, size_t len) +{ + memory_cleanse(ptr, len); +} + +extern "C" int qbit_mlkem_has_avx2(void) +{ + return !t_force_portable && g_cpu_x86_native.load(std::memory_order_relaxed) ? 1 : 0; +} + +extern "C" int qbit_mlkem_has_neon(void) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + // NEON is mandatory on AArch64. + return !t_force_portable ? 1 : 0; +#else + return 0; +#endif +} + +namespace mlkem { + +Error KeyGen(std::span seed64, PublicKey& ek, DecapsulationKey& dk) noexcept +{ + BeginOperation(); + const int result{Injected(Operation::KEYGEN, qbit_mlkem1024_keypair_derand(ek.data(), dk.Bytes().data(), seed64.data()))}; + const Error error{MapResult(Operation::KEYGEN, result)}; + if (error != Error::NONE) { + dk.Clear(); + memory_cleanse(ek.data(), ek.size()); + } + return error; +} + +Error CheckPublicKey(std::span ek) noexcept +{ + BeginOperation(); + return MapResult(Operation::CHECK_PUBLIC_KEY, Injected(Operation::CHECK_PUBLIC_KEY, qbit_mlkem1024_check_pk(ek.data()))); +} + +Error Encaps(std::span ek, std::span coins32, + Ciphertext& ct, SharedSecret& ss) noexcept +{ + BeginOperation(); + const int result{Injected(Operation::ENCAPS, qbit_mlkem1024_enc_derand(ct.data(), ss.Bytes().data(), ek.data(), coins32.data()))}; + const Error error{MapResult(Operation::ENCAPS, result)}; + if (error != Error::NONE) { + ss.Clear(); + memory_cleanse(ct.data(), ct.size()); + } + return error; +} + +Error Decaps(const DecapsulationKey& dk, std::span ct, SharedSecret& ss) noexcept +{ + BeginOperation(); + const int result{Injected(Operation::DECAPS, qbit_mlkem1024_dec(ss.Bytes().data(), ct.data(), dk.Bytes().data()))}; + const Error error{MapResult(Operation::DECAPS, result)}; + if (error != Error::NONE) ss.Clear(); + return error; +} + +void InitializeRuntime(bool force_portable) noexcept +{ + DetectCapabilities(); + g_force_portable.store(force_portable); +} + +BackendNames GetBackendNames() noexcept +{ + BeginOperation(); + return { + .arith = ActiveBackend(qbit_mlkem_compiled_arith_backend()), + .keccak = ActiveBackend(qbit_mlkem_compiled_keccak_backend()), + }; +} + +ForcePortableForTesting::ForcePortableForTesting() noexcept + : m_previous{g_force_portable.exchange(true)} +{ +} + +ForcePortableForTesting::~ForcePortableForTesting() +{ + g_force_portable.store(m_previous); +} + +InjectResultForTesting::InjectResultForTesting(Operation op, int upstream_result) noexcept + : m_op{op}, m_previous{g_injected_results[static_cast(op)].exchange(upstream_result)} +{ +} + +InjectResultForTesting::~InjectResultForTesting() +{ + g_injected_results[static_cast(m_op)].store(m_previous); +} + +} // namespace mlkem diff --git a/src/crypto/mlkem.h b/src/crypto/mlkem.h new file mode 100644 index 0000000000..92b7e2fff9 --- /dev/null +++ b/src/crypto/mlkem.h @@ -0,0 +1,153 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_H +#define QBIT_CRYPTO_MLKEM_H + +#include + +#include +#include +#include +#include +#include + +/** + * ML-KEM-1024 (FIPS 203), backed by the vendored mlkem-native library. + * + * Native code runs where it is supported and the CPU allows it (x86_64 with + * AVX2, AArch64), and portable C everywhere else. The library is built into + * bitcoin_node only. + * + * Fixed-size spans make wrong-sized calls impossible; callers check network + * lengths before constructing them. Entropy is the caller's job: key + * generation takes two independent 32-byte draws, encapsulation one. + */ +namespace mlkem { +inline constexpr size_t PUBLIC_KEY_BYTES = 1568; +inline constexpr size_t SECRET_KEY_BYTES = 3168; +inline constexpr size_t CIPHERTEXT_BYTES = 1568; +inline constexpr size_t SHARED_SECRET_BYTES = 32; +inline constexpr size_t KEYGEN_SEED_BYTES = 64; +inline constexpr size_t ENCAPS_COINS_BYTES = 32; + +/** Fixed-size secret bytes, wiped with memory_cleanse on destruction, Clear() and move-from. */ +template +class Secret +{ + std::array m_bytes{}; + +public: + Secret() noexcept = default; + ~Secret() { Clear(); } + Secret(const Secret&) = delete; + Secret& operator=(const Secret&) = delete; + Secret(Secret&& other) noexcept : m_bytes{other.m_bytes} { other.Clear(); } + Secret& operator=(Secret&& other) noexcept + { + if (this != &other) { + m_bytes = other.m_bytes; + other.Clear(); + } + return *this; + } + std::span Bytes() noexcept { return m_bytes; } + std::span Bytes() const noexcept { return m_bytes; } + void Clear() noexcept { memory_cleanse(m_bytes.data(), m_bytes.size()); } +}; + +using PublicKey = std::array; +using Ciphertext = std::array; +using DecapsulationKey = Secret; +using SharedSecret = Secret; + +enum class Error : uint8_t { NONE = 0, INVALID_PUBLIC_KEY = 1, INVALID_SECRET_KEY = 2, INTERNAL = 3 }; + +/** + * Deterministic key generation (FIPS 203 ML-KEM.KeyGen_internal) from + * seed64 = d || z. On failure, ek and dk are cleared. + */ +[[nodiscard]] Error KeyGen(std::span seed64, PublicKey& ek, DecapsulationKey& dk) noexcept; + +/** The FIPS 203 section 7.2 modulus check: INVALID_PUBLIC_KEY if a coefficient is not below q. */ +[[nodiscard]] Error CheckPublicKey(std::span ek) noexcept; + +/** + * Deterministic encapsulation (FIPS 203 ML-KEM.Encaps_internal), including the + * modulus check of ek. On failure, ct and ss are cleared. + */ +[[nodiscard]] Error Encaps(std::span ek, std::span coins32, + Ciphertext& ct, SharedSecret& ss) noexcept; + +/** + * Decapsulation (FIPS 203 ML-KEM.Decaps), including the section 7.3 hash + * check of dk (INVALID_SECRET_KEY). A corrupted ciphertext of the right + * length is not an error: it yields the implicit-rejection secret, which + * differs from the encapsulated one. On failure, ss is cleared. + */ +[[nodiscard]] Error Decaps(const DecapsulationKey& dk, std::span ct, + SharedSecret& ss) noexcept; + +struct BackendNames { + std::string_view arith; //!< "x86_64-avx2", "aarch64-neon" or "portable" + std::string_view keccak; //!< "x86_64-avx2", "aarch64" or "portable" +}; + +/** + * Detect CPU features and set the portable override. Call once at startup, + * before networking starts. Operations work without it, with auto-detection. + */ +void InitializeRuntime(bool force_portable) noexcept; + +/** The implementation that runs now: native only if compiled in, supported by the CPU and not overridden. */ +BackendNames GetBackendNames() noexcept; + +/** + * Forces portable C while alive, then restores the previous override (RAII). + * Each operation reads the override once, at entry, so one that is already + * running finishes on the backend it started with. Test code only. + */ +class ForcePortableForTesting +{ + bool m_previous; + +public: + ForcePortableForTesting() noexcept; + ~ForcePortableForTesting(); + ForcePortableForTesting(const ForcePortableForTesting&) = delete; + ForcePortableForTesting& operator=(const ForcePortableForTesting&) = delete; +}; + +/** mlkem-native's error codes (mlkem/mlkem_native.h), for InjectResultForTesting. */ +namespace upstream { +inline constexpr int ERR_FAIL{-1}; +inline constexpr int ERR_OUT_OF_MEMORY{-2}; +inline constexpr int ERR_RNG_FAIL{-3}; +inline constexpr int ERR_INVALID_PK{-4}; +inline constexpr int ERR_INVALID_SK{-5}; +inline constexpr int ERR_PCT_FAIL{-6}; +} // namespace upstream + +enum class Operation : uint8_t { KEYGEN, CHECK_PUBLIC_KEY, ENCAPS, DECAPS }; + +/** + * Test-only (RAII): while alive, `op` behaves as if mlkem-native had returned + * `upstream_result` (non-zero) after writing its outputs. This drives the + * wrapper's error mapping and output clearing, and lets callers test their + * handling of internal errors. Single-threaded test code only. + */ +class InjectResultForTesting +{ + Operation m_op; + int m_previous; + +public: + InjectResultForTesting(Operation op, int upstream_result) noexcept; + ~InjectResultForTesting(); + InjectResultForTesting(const InjectResultForTesting&) = delete; + InjectResultForTesting& operator=(const InjectResultForTesting&) = delete; +}; +} // namespace mlkem + +#endif // QBIT_CRYPTO_MLKEM_H diff --git a/src/crypto/mlkem_arith_backend.h b/src/crypto/mlkem_arith_backend.h new file mode 100644 index 0000000000..8d9a49625d --- /dev/null +++ b/src/crypto/mlkem_arith_backend.h @@ -0,0 +1,29 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_ARITH_BACKEND_H +#define QBIT_CRYPTO_MLKEM_ARITH_BACKEND_H + +// mlkem-native arithmetic backend (MLK_CONFIG_ARITH_BACKEND_FILE), used only +// when crypto/mlkem_config.h enables native code. +// +// Upstream's default selector, native/meta.h, picks the x86_64 backend only +// when the compiler itself targets AVX2, which baseline release builds do not. +// qbit includes the backend unconditionally instead: it is assembly plus +// constant tables, and every entry point asks mlk_sys_check_capability() +// before running, so CPUs without AVX2 take the portable C path. qbit answers +// MLK_SYS_CAP_X86_64_AVX2 for every extension the assembly executes (AVX2, +// SSSE3, SSE4.1, POPCNT, BMI2), not only the AVX2 its name says. + +#if defined(MLK_SYS_X86_64) +#define QBIT_MLKEM_ARITH_BACKEND_NAME "x86_64-avx2" +#include +#elif defined(MLK_SYS_AARCH64) +#define QBIT_MLKEM_ARITH_BACKEND_NAME "aarch64-neon" +#include +#else +#error "No mlkem-native arithmetic backend for this target." +#endif + +#endif // QBIT_CRYPTO_MLKEM_ARITH_BACKEND_H diff --git a/src/crypto/mlkem_backend.c b/src/crypto/mlkem_backend.c new file mode 100644 index 0000000000..933261e8ab --- /dev/null +++ b/src/crypto/mlkem_backend.c @@ -0,0 +1,29 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +// Reports which native backends mlkem-native was compiled with. This file is +// built with the same configuration and flags as mlkem-native's single +// compilation unit, so the backend headers it sees are the ones compiled in. + +#include + +#include + +const char* qbit_mlkem_compiled_arith_backend(void) +{ +#if defined(QBIT_MLKEM_ARITH_BACKEND_NAME) + return QBIT_MLKEM_ARITH_BACKEND_NAME; +#else + return "portable"; +#endif +} + +const char* qbit_mlkem_compiled_keccak_backend(void) +{ +#if defined(QBIT_MLKEM_KECCAK_BACKEND_NAME) + return QBIT_MLKEM_KECCAK_BACKEND_NAME; +#else + return "portable"; +#endif +} diff --git a/src/crypto/mlkem_config.h b/src/crypto/mlkem_config.h new file mode 100644 index 0000000000..488db496bf --- /dev/null +++ b/src/crypto/mlkem_config.h @@ -0,0 +1,89 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_CONFIG_H +#define QBIT_CRYPTO_MLKEM_CONFIG_H + +// qbit's configuration of the vendored mlkem-native library, selected with +// MLK_CONFIG_FILE in place of upstream's mlkem/mlkem_native_config.h, which +// documents every option used here. cmake/mlkem-native.cmake decides whether +// native code may be built and passes that decision in: +// +// QBIT_MLKEM_NATIVE build the native backends for this target (x86_64 with +// the SysV ABI, or AArch64 with NEON on Apple: AArch64 +// ELF targets stay portable because the assembly would +// strip their BTI and PAC marking); +// MLK_CONFIG_NO_ASM no assembly at all (MSVC and other toolchains without +// GNU assembler support). +// +// Neither set means portable C with inline-assembly value barriers. + +#define MLK_CONFIG_PARAMETER_SET 1024 +#define MLK_CONFIG_NAMESPACE_PREFIX qbit_mlkem1024 +// Only the deterministic APIs exist: callers supply all entropy. +#define MLK_CONFIG_NO_RANDOMIZED_API + +#if defined(MLK_BUILD_INTERNAL) + +#define MLK_CONFIG_CUSTOM_ZEROIZE +#define MLK_CONFIG_CUSTOM_CAPABILITY_FUNC + +// After the options sys.h reads, before the code that needs its definitions. +#include + +#if defined(QBIT_MLKEM_NATIVE) +#if defined(MLK_CONFIG_NO_ASM) +#error "QBIT_MLKEM_NATIVE and MLK_CONFIG_NO_ASM are both set; cmake/mlkem-native.cmake sets at most one." +#endif +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#error "MemorySanitizer cannot see writes made by assembly; configure with WITH_MLKEM_NATIVE=AUTO or OFF." +#endif +#endif +#if defined(MLK_SYS_AARCH64) && defined(__ELF__) +#error "mlkem-native AArch64 assembly has no BTI landing pads or GNU property note; AArch64 ELF builds are portable only." +#endif +// Enable a backend only where qbit selects and tests one. Never define these +// to 0: upstream checks whether they are defined. +#if (defined(MLK_SYS_X86_64) && defined(MLK_SYSV_ABI_SUPPORTED)) || \ + (defined(MLK_SYS_AARCH64) && defined(MLK_SYS_AARCH64_NEON) && defined(MLK_SYS_APPLE)) +#define MLK_CONFIG_USE_NATIVE_BACKEND_ARITH +#define MLK_CONFIG_USE_NATIVE_BACKEND_FIPS202 +#define MLK_CONFIG_ARITH_BACKEND_FILE "crypto/mlkem_arith_backend.h" +#define MLK_CONFIG_FIPS202_BACKEND_FILE "crypto/mlkem_fips202_backend.h" +#else +#error "QBIT_MLKEM_NATIVE is set, but qbit has no native ML-KEM backend for this target; cmake/mlkem-native.cmake and this header disagree." +#endif +#endif // QBIT_MLKEM_NATIVE + +#if !defined(__ASSEMBLER__) +#include + +static MLK_INLINE void mlk_zeroize(void* ptr, size_t len) +{ + qbit_mlkem_zeroize(ptr, len); +} + +// Backends call this 80 to 100 times per operation, so it reads only state +// that crypto/mlkem.cpp cached before the operation started. Capabilities qbit +// does not use, MLK_SYS_CAP_AARCH64_SHA3 included, are never available. +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_sys_check_capability(mlk_sys_cap cap) +{ +#if defined(MLK_SYS_X86_64) + // Upstream gates its whole x86_64 backend on this one capability, so the + // answer covers every extension that assembly executes, not only AVX2. + if (cap == MLK_SYS_CAP_X86_64_AVX2) return qbit_mlkem_has_avx2() ? 1 : 0; +#endif +#if defined(MLK_SYS_AARCH64) + if (cap == MLK_SYS_CAP_AARCH64_NEON) return qbit_mlkem_has_neon() ? 1 : 0; +#endif + (void)cap; + return 0; +} +#endif // !__ASSEMBLER__ + +#endif // MLK_BUILD_INTERNAL + +#endif // QBIT_CRYPTO_MLKEM_CONFIG_H diff --git a/src/crypto/mlkem_fips202_backend.h b/src/crypto/mlkem_fips202_backend.h new file mode 100644 index 0000000000..26138ce5db --- /dev/null +++ b/src/crypto/mlkem_fips202_backend.h @@ -0,0 +1,55 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_FIPS202_BACKEND_H +#define QBIT_CRYPTO_MLKEM_FIPS202_BACKEND_H + +// mlkem-native Keccak backend (MLK_CONFIG_FIPS202_BACKEND_FILE), used only +// when crypto/mlkem_config.h enables native code. See +// crypto/mlkem_arith_backend.h for why the backend is chosen here rather than +// by upstream's fips202/native/auto.h. + +#if defined(MLK_SYS_X86_64) +#define QBIT_MLKEM_KECCAK_BACKEND_NAME "x86_64-avx2" +#include + +#elif defined(MLK_SYS_AARCH64) +#define QBIT_MLKEM_KECCAK_BACKEND_NAME "aarch64" +// The Keccak upstream picks for AArch64 without the SHA3 extension, used on +// Apple too even though Apple CPUs have it, so that every native AArch64 build +// runs the same code. (Native AArch64 is Apple-only for now; see +// crypto/mlkem_config.h.) +#include + +// Single-lane Keccak. This mirrors mlkem/src/fips202/native/aarch64/x1_scalar.h +// of mlkem-native v2.0.0 (d1b2fe782888bdb761a50336012923180be7f502): the same +// macros and the same proved assembly routine. The one difference is the +// capability check. Upstream's x1 entry point runs its scalar assembly +// unconditionally, so the portable override could not reach it, and +// mlkem::GetBackendNames() would claim portable Keccak while assembly ran. +// Here it falls back to C whenever the AArch64 capability is off. On every +// library update, diff upstream's x1_scalar.h against this block (see +// doc/subtrees/mlkem-native.md). +#define MLK_USE_NATIVE_FIPS202_X1 +#define MLK_FIPS202_AARCH64_NEED_X1_SCALAR +#if !defined(__ASSEMBLER__) +#include +#include +MLK_MUST_CHECK_RETURN_VALUE +static MLK_INLINE int mlk_keccak_f1600_x1_native(uint64_t* state) +{ + if (!mlk_sys_check_capability(MLK_SYS_CAP_AARCH64_NEON)) { + return MLK_NATIVE_FUNC_FALLBACK; + } + + mlk_keccak_f1600_x1_scalar_aarch64_asm(state, mlk_keccakf1600_round_constants); + return MLK_NATIVE_FUNC_SUCCESS; +} +#endif // !__ASSEMBLER__ + +#else +#error "No mlkem-native Keccak backend for this target." +#endif + +#endif // QBIT_CRYPTO_MLKEM_FIPS202_BACKEND_H diff --git a/src/crypto/mlkem_shim.h b/src/crypto/mlkem_shim.h new file mode 100644 index 0000000000..a7824fa339 --- /dev/null +++ b/src/crypto/mlkem_shim.h @@ -0,0 +1,44 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#ifndef QBIT_CRYPTO_MLKEM_SHIM_H +#define QBIT_CRYPTO_MLKEM_SHIM_H + +// C ABI between mlkem-native, which is C compiled with crypto/mlkem_config.h, +// and qbit's C++ code. mlkem-native's assembly includes the configuration too, +// so the declarations are hidden from the assembler. + +#if !defined(__ASSEMBLER__) +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Implemented in C++, in crypto/mlkem.cpp. + +/** Wipe len bytes at ptr. Forwards to memory_cleanse. */ +void qbit_mlkem_zeroize(void* ptr, size_t len); +/** + * 1 if mlkem-native may run its x86_64 assembly now, else 0. Besides AVX2 that + * assembly needs SSSE3, SSE4.1, POPCNT and BMI2 (cpu_features::HasMlkemX86Native). + * Reads cached state only. + */ +int qbit_mlkem_has_avx2(void); +/** 1 if mlkem-native may run its AArch64 code now, else 0. Reads cached state only. */ +int qbit_mlkem_has_neon(void); + +// Implemented in C, in crypto/mlkem_backend.c, with mlkem-native's configuration. + +/** Name of the native arithmetic backend compiled in, or "portable". */ +const char* qbit_mlkem_compiled_arith_backend(void); +/** Name of the native Keccak backend compiled in, or "portable". */ +const char* qbit_mlkem_compiled_keccak_backend(void); + +#ifdef __cplusplus +} +#endif +#endif // !__ASSEMBLER__ + +#endif // QBIT_CRYPTO_MLKEM_SHIM_H diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index b08c3d0f2e..204f378b34 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -67,6 +67,7 @@ add_executable(test_bitcoin miner_tests.cpp miniminer_tests.cpp miniscript_tests.cpp + mlkem_tests.cpp minisketch_tests.cpp multisig_tests.cpp net_peer_connection_tests.cpp @@ -158,6 +159,7 @@ target_json_data_sources(test_bitcoin data/key_io_invalid.json data/key_io_valid.json data/mainnet_launch_difficulty.json + data/mlkem1024_vectors.json data/p2mr_cross_profile_vectors.json data/p2mr_datapqchash_vectors.json data/p2mr_pqc_witness_vectors.json diff --git a/src/test/data/mlkem1024_vectors.json b/src/test/data/mlkem1024_vectors.json new file mode 100644 index 0000000000..cbe5eceec4 --- /dev/null +++ b/src/test/data/mlkem1024_vectors.json @@ -0,0 +1,144 @@ +{ + "_comment": [ + "ML-KEM-1024 (FIPS 203) vectors for src/test/mlkem_tests.cpp.", + "acvp: test cases copied verbatim from NIST ACVP-Server gen-val/json-files at commit 975de31eb83d87039ec88934fdc47d8c312b892d,", + "ML-KEM-keyGen-FIPS203/internalProjection.json (sha256 d7a62a2c3476957f56dd8d24f9004ea6776ccfe995ffe71a65bb9506dc9c7b1b) and", + "ML-KEM-encapDecap-FIPS203/internalProjection.json (sha256 a556952ce869bb89c3a3196a701dad89647c193a34c86eafb61a9d710d5b810f),", + "parameter set ML-KEM-1024, selected by tcId. Fields not used by the tests are omitted.", + "cctv_modulus: C2SP CCTV ML-KEM/modulus/ML-KEM-1024.txt.gz at commit 6ec11f7d0ef5ebab45ff21ca7e06e643e5182728 lists 1040 invalid", + "encapsulation keys that share every coefficient but one. base_ek is that shared key with every coefficient in range;", + "the tests rebuild all 1040 keys from it and check that the SHA-256 of their hex lines equals sha256_of_txt, the SHA-256 of", + "the decompressed ML-KEM-1024.txt." + ], + "acvp": { + "keyGen": [ + { + "tcId": 51, + "d": "F3A706FAF090C03DB506863AB0B20BD8A1627956318E88C67EB875E8E7266009", + "z": "35D2BC43DD1CC879F765BF2A0C5E297889DDE910E57E2BB0EAE417B90AB7A275", + "ek": "8D0923CA8A2DA2B4146EC25321122B8A5AA8AFE0C03415273008A46EE83031E98AAAA125ABC75D3B30322560C197E75DD0E48A348099F7B2144D7B8A8660A4A97BCF19C0583BD9BB2123033CD7BB5A14B08B817831A673A28170F5F6443C0551913A327CBA18C3A053C4040250403B70AB9588832403AA0FC37665E04980FE1602E7D2715D9CBC00515DF432A4F5B32B3BC92AE3F31700166D498123E94576509B712B18491B1435EE7AB7AEB1AD30D72348C3CC083ABE24A8B12097BF32F792476288EECC3BF630ADCDAC6ACA7950D9839501A448500742BAE37F109203A809B2B960A307E25347A32C3EAB79288173A878789B296E9E8C1C28C5BB3AC472601C9765F7B77225A810C7B85370BEF4A5B079D2015ADA54236B8F33840675F9B2EB427A1B5974CD5C61B24010886C5A7BDA5BBED974AF7217F3338AD719CB308A8BCB1B6D6ED2A1643736C29095E8A8452A3A36C7BB5AE58CBFDC61529466A90F454ED6895B0861083DD1371999B2F559A3A487CF59A074FB49215EA6A6BE656F9AF17B121A2447CB7985590E9738842B899BA57AC311810AE2D9794F37483DD6BCCB64AF6D56588AE94665961C025C3AA2861974C236BCA4BD8FF5509F7AB774593E7C5549E57C2F18D15C0515094AD9A0DFAA0601E524F8231156B627BB25A0DAE04DACD0A66C041CEF400583FC13BAE640291A39A5C5CA8BCA1AD5C683CDD8290891A76940817DD8C9F52678780548E37A05806600801426DBB950C3B2BA34E24CC77864DD91B39F1408C716A69DF63342854E50A245FB50977B9410DED2C93F86B1F9D5A78B87BF81E51CA620A7E8566B19AB700964A40E3266415228D432156E5CBDF52364A90483A55C39B3FB16FC7465A3F8AC801B70B9FB28B583444BA5C1A73722D417A9D6D9B7DEB08BC6B330FF27CF61AB8831E27758C64AF3B12150CB7B33ABC29858106D63686D8762459ABF9413850AE53ED6313F76F83D0FB8AB34374E7DF693E4A1B3E5A8AD0CE820AFE1CF401ACDE650A8101B0946022D52178E19613C42B88B07CC04EAA81DFB28AC9DC076236B67219A30F8F945DD57BD2F335C52D59372308D38993467DB53DA3382B74867B616481BD0091A2232C1116DC88A589DB9107224A681008C67C589186A6929549BEEF92253DB02B0C8AA9F9C875A670266C72BCBDB4F5625043703C1A0457395832E4C335180462ED2220C59E7361903C107D85457F6CD82EB820D0855D97675C2E0151CDB73C2885DDB7849D74541580124E890116A65BC068093B57914E20C937C60A3EB25576F1A976A9583839B672144CD4A45C3477A45C29B4E0BC2BDBD206585C9B7A7741C8B6B5793A92797A15AE7A5B73A74B2971463634BA52AA792AF05530730B6D0A89A346156B733677932BD36593A7496130CC458DCC5CA987C21960604EC8A8C5396056680CBF3F1AAC4F401AA5029FB2150434BB4706C31A2D54E4297939FA7C9C6F85700613CEB65C7F03AC56EB86E2D27CCC6DCB7B9394DCDB942FF222D86958A996C0CB6A8A44F97A70441C95FA71250116EEC20863C0B5A643458788AB001F8869D909922F51EE547A1E889255B3A0599C65842E5AB8D73872F053BC62392EA53896D328102D460BF1609583C22C3B43780EC6DAD0319EB4A5A65B4756C3CB40EAA935183BF8BD46ABE76BA46E199103A5313C3235F49C915E097BCA804DE680781D8365731BEAC6789A9203FB8787C4C070E00A13A6722A66A28236DB179825653D33CCF898B72C6B8450D97AFD3276BB13340519CBEDA708D12A858F54C49F4547195B7788A9150B2649E36AA394121926D568A488B16D3557B2A32AF57D11FC3373F80A28C0723273D362502E7C428AB44D3CBABF9EA585FD1BD0C9846556A1E196B78CF951592984A0A8487A78C2317D7ACA4118E1049750A0788F0D66AD9E48E34731130ABA0B427360A856D96D80B3F028FDD3ABA9035C10106BA1C0934BED36C6D7C7434249654EA89FC22137F4AB903653B75FB25B6F01635E6CC7D39CF1508690562826B49B6FFC59E0DD35022E541F8BA0D304AA5B4E20606907C424395666C54ABC2B8FB009847C86317685000C231215C8C15945860F6A85DDB98A8C3A527F2749D3C027E694E8F0B0F0FA454913AADB635AADD452F7128BF7752569669A8B93290EB92E78F6ADFF23E89F57F3890753B51F12F3F3A8A654E677847", + "dk": "6CBBB491D7B3A3439F53C9027BD727A0E1C78004B4CDB66760E50F1BF97FFF0139F82862F1F3178A077B6AF77735CB802E91316BC207B9F1485FF7A868891E003BB9C6558BE6052C0826AF1CB5836B94C7D5D714EEFC7CFD983E9884B71348499DCA05DEF33B49D1C14BBBB95AB195BD72219B18BD0C704275CC94AB06653BBB9E0EEC74181903A0081AAE068B81EBAE8925335B8134B1480A60F758282C2F011B909641266774513A19090BC672EB3A0708CB577F134151C155C1E560D9B335A35A4D75F223502752CD250C9A7778EFC76A52709813A24679B987A6172F1714641EA6902D364F96357E92B44275EBCCE7F6207BB0A215E98B3AA92DAA46A39AD9132F3874C6239391E716BA600E42BC7084425051F865756A89DAB46A76104E183A12031B99BE86257B583C2E88104FEC3BFCAA50F238BEEC20ABF31219685885F2C3CE2F80C09B874E73A8712D35B8EE819D84B1C59A37947C0B9E25F066EE030ED28172FD5914B562408BA9B9DE9BAB2B95A4F30C22ECE458E133371A693D9458058265A75AD6CA5707617B38A1B6F396FB199CC1C77100741DE37500F1BBB10B9204EEB7C3F5B91A1CC91B6D4072AF410957358ACB0625488B9F6A9915758B7CC9B4AFE7D5AD6A95231A7A711B3A8996E23AFB3695B45B6D919C771D7685D90A7550A9B5B1E7538A365451A5A815009E49326C6541CAF0BA685056BF82EA4AB504161B555568583802800554A9480602437B0B9677FAC638A2508154A658E0623C62666E078D21A5A545A818D66C9547B9749458B735A4AD51812950E8A625A1B2A753ABBBA16415006300AD32D34C8B0B3B2AF4D1CFBDE08B4F06C70DE4BA28FB3763745CCE133DD2E488F89A51E7A586A86AC928867B0541438AF5645DE3211AA94ACA4A1095896EBC363C374A85565C1EBA030B5EA21B9B8A140E131ED07580D5B8590446988DA65939A823A84661B26CA12461AFE373519F43C13987A96A9069CD6B9177B205B25B5197DBCDE66C6457892C1DF96FE50AB7EF3B3DAB0A87436B3451476C44AA22AEF918AE5A1A8AE46920263032E3BC5DEBCFEE43B8AA290CEDF06966939EE6CC7C6AF83CB9D90CC889AB8B5A3D4C224E73916AE057BF5455528CACBED31B554043C6B0EA196854461D2644461470EFC55080E326FED49D62B5A4BB06B14F505EB4486C9A1B033B927836DA656C2475E5B8B823986DCA99B298512D94B77FE0588DBB96BA53A67369B82C2BEAB2CCC0103E549317A99DD97B598FA0A506184EB55C96B17302533C7EC3334E3CEC2BBB22BE0F89442840884FAB4A766A41BC8061D4ABCAC5190C729309AC6151BD434B818BC1E7D0CD435C9CA6DC7182228FF93867134A5CC65C6167C4A8BB830E7FEB863C15BF87B259EB0518FF5248792A625F37C589589799746D36197DC63AABD134A00EF1B49C79A2DF67B28E6B6E989CC9D0577EA5AA1B2FE64B322C7A8B241D503523D875CCDC479D28E3C01BD8383DCB2B684281E6949171F518EA97009237B5CD42AC8FC8A571716EB784B336529DB67728499988B73B80EF5C6D33F3B42ADA7A0A36A7C6A8A0DFB804B3D7BB05803957B913827A16F52ACDD7650EE2971A85DC21F6A94A5B31988E4438A9C1733AB8C48C32653FE800BF81BA66658CD2001ACF3A83E3A0426AB6C3B2600A53621277A43D723AB96758B120F120248C15EDF4294FC38F96EC490BD52094655841D9355038A379D8B666875171F39218E21D37023F5108B6D15389F07C1241E15071C43CD7D160EB5341B719C2055424EF6CBF66815F84943C9481CD1DE685A74C5B9DB37A0713952FD4A67BEC8CA7496F6C9C5FB84A8EAFE68D54654C6BAC9C3AB712C2BC32C0CAB072C18DE3CABBD9224DFAA305DFDB3F85019D7B988577B40F27514822DB42D87489502C53CCE6A095B486D9143D6D160B1C7313BF288913657698117EB145C0C9C55D66C15BF1A2B268F6A5FA259139551ABFE3468F1CC49C01B4232543DEF0BF9AE503B998536310751A8BC302B4B4440B835233797C967E7F9308155B27BA63C65E08BDD1946828196B30C042CDA0B56E4008215AC51F373604F5930BB66297C32BE41A162CA21471CC86657B5CB5307D89091DA9A5137F1C1433AB8AC1D59DE421C7C17A883EAB99BA4714F841998D0923CA8A2DA2B4146EC25321122B8A5AA8AFE0C03415273008A46EE83031E98AAAA125ABC75D3B30322560C197E75DD0E48A348099F7B2144D7B8A8660A4A97BCF19C0583BD9BB2123033CD7BB5A14B08B817831A673A28170F5F6443C0551913A327CBA18C3A053C4040250403B70AB9588832403AA0FC37665E04980FE1602E7D2715D9CBC00515DF432A4F5B32B3BC92AE3F31700166D498123E94576509B712B18491B1435EE7AB7AEB1AD30D72348C3CC083ABE24A8B12097BF32F792476288EECC3BF630ADCDAC6ACA7950D9839501A448500742BAE37F109203A809B2B960A307E25347A32C3EAB79288173A878789B296E9E8C1C28C5BB3AC472601C9765F7B77225A810C7B85370BEF4A5B079D2015ADA54236B8F33840675F9B2EB427A1B5974CD5C61B24010886C5A7BDA5BBED974AF7217F3338AD719CB308A8BCB1B6D6ED2A1643736C29095E8A8452A3A36C7BB5AE58CBFDC61529466A90F454ED6895B0861083DD1371999B2F559A3A487CF59A074FB49215EA6A6BE656F9AF17B121A2447CB7985590E9738842B899BA57AC311810AE2D9794F37483DD6BCCB64AF6D56588AE94665961C025C3AA2861974C236BCA4BD8FF5509F7AB774593E7C5549E57C2F18D15C0515094AD9A0DFAA0601E524F8231156B627BB25A0DAE04DACD0A66C041CEF400583FC13BAE640291A39A5C5CA8BCA1AD5C683CDD8290891A76940817DD8C9F52678780548E37A05806600801426DBB950C3B2BA34E24CC77864DD91B39F1408C716A69DF63342854E50A245FB50977B9410DED2C93F86B1F9D5A78B87BF81E51CA620A7E8566B19AB700964A40E3266415228D432156E5CBDF52364A90483A55C39B3FB16FC7465A3F8AC801B70B9FB28B583444BA5C1A73722D417A9D6D9B7DEB08BC6B330FF27CF61AB8831E27758C64AF3B12150CB7B33ABC29858106D63686D8762459ABF9413850AE53ED6313F76F83D0FB8AB34374E7DF693E4A1B3E5A8AD0CE820AFE1CF401ACDE650A8101B0946022D52178E19613C42B88B07CC04EAA81DFB28AC9DC076236B67219A30F8F945DD57BD2F335C52D59372308D38993467DB53DA3382B74867B616481BD0091A2232C1116DC88A589DB9107224A681008C67C589186A6929549BEEF92253DB02B0C8AA9F9C875A670266C72BCBDB4F5625043703C1A0457395832E4C335180462ED2220C59E7361903C107D85457F6CD82EB820D0855D97675C2E0151CDB73C2885DDB7849D74541580124E890116A65BC068093B57914E20C937C60A3EB25576F1A976A9583839B672144CD4A45C3477A45C29B4E0BC2BDBD206585C9B7A7741C8B6B5793A92797A15AE7A5B73A74B2971463634BA52AA792AF05530730B6D0A89A346156B733677932BD36593A7496130CC458DCC5CA987C21960604EC8A8C5396056680CBF3F1AAC4F401AA5029FB2150434BB4706C31A2D54E4297939FA7C9C6F85700613CEB65C7F03AC56EB86E2D27CCC6DCB7B9394DCDB942FF222D86958A996C0CB6A8A44F97A70441C95FA71250116EEC20863C0B5A643458788AB001F8869D909922F51EE547A1E889255B3A0599C65842E5AB8D73872F053BC62392EA53896D328102D460BF1609583C22C3B43780EC6DAD0319EB4A5A65B4756C3CB40EAA935183BF8BD46ABE76BA46E199103A5313C3235F49C915E097BCA804DE680781D8365731BEAC6789A9203FB8787C4C070E00A13A6722A66A28236DB179825653D33CCF898B72C6B8450D97AFD3276BB13340519CBEDA708D12A858F54C49F4547195B7788A9150B2649E36AA394121926D568A488B16D3557B2A32AF57D11FC3373F80A28C0723273D362502E7C428AB44D3CBABF9EA585FD1BD0C9846556A1E196B78CF951592984A0A8487A78C2317D7ACA4118E1049750A0788F0D66AD9E48E34731130ABA0B427360A856D96D80B3F028FDD3ABA9035C10106BA1C0934BED36C6D7C7434249654EA89FC22137F4AB903653B75FB25B6F01635E6CC7D39CF1508690562826B49B6FFC59E0DD35022E541F8BA0D304AA5B4E20606907C424395666C54ABC2B8FB009847C86317685000C231215C8C15945860F6A85DDB98A8C3A527F2749D3C027E694E8F0B0F0FA454913AADB635AADD452F7128BF7752569669A8B93290EB92E78F6ADFF23E89F57F3890753B51F12F3F3A8A654E6778479370FE5B05DDC92C939F62CBDE4C0FEA36F45CD20C5748CF3AC891A4C260449635D2BC43DD1CC879F765BF2A0C5E297889DDE910E57E2BB0EAE417B90AB7A275" + }, + { + "tcId": 52, + "d": "52F2DF12173088140CA73333D1DB4341949BE452049028EF0DD904BC7E0DDEF7", + "z": "CC96DE9E1F0506BF490A154A6DB46BD2BD1413C3D1DC163190FF171EF0D11E22", + "ek": "88C29C2A91CE80B13EEBE182AA709A1694A456B139DEB1977D531AD2722AC7D2B356AA2684D767C60012B1A2345A69AC52C7A814858E8C741A80C4B5E942CEEE0045B957748E2847EA835E1FE17C70456DBEA13FCEE0384BF431225C8A6B499624F086BF71518C8C269640B436E7783057372825768670B2478C96C4A1A2ADCB1F83A057263362476197C5D878D76B025F83AC802CCC02296A99296CF6B23C5617AA95387C2B0071C23AB7BF614371BC7001E29D7109057DC03053FB38C0084C48890612369C649C0BA3C2603E377C84D15B6DB6BB6E9513ADA9A47210861AA055D14683F0556729192CAEA8AE9A538FF810596D64BC0964B380CBC2F5D706A0A8135C1B87C3480C6D54CFC28A53C339666E24213D33A62EC872D803103987A36256840DD98DA8BC17ED42C34F5C008F6C678C748F3F7404D17B46F29012F079A46A0ABF77C04538254ED4B6BDEA929724AA60014674C427C55B1B0192CB284ED02194F9615417B08A9580DCA78944F43094E48AEBBC8BF0126310D9AA79530457E04DBA9B0D02B045AB28391F923C25CAA69AD038B4586350613D9AE22896F2979D56BBAFDAA294041AA42C21B9AC59E41CBEDBC33AD6996C3451B7DF5B9AE2D003F5092B91847290336A62E804B5F167D7CC4E31F40F6C640C10EB99D7F5519C80147B77843CD083B3B5C03718B1A7EA2DBE25ADBD56762E9150B46B6B4E82B769C384CAB39B3610070C6B6E5981C3B182105981A50567514CA15EB44620C231740FEB6BCBF4980F7B1824A5602A982A68BC8961BBC441B111C93B2374EC88FEF30D4931AB0CE3B5BEA680B1099CFA79B3FFF2B56BB3031A8C07519C7DF4F75A8C887C6A1B7249A700717B2798C02F0EA04E720752D8323376E1635331B07435CDD5BA6D7228281612BF060C694BAB6A00A8700977518BBAC8195853FDD34D44E29FEA7286DCFABDF47C9B2F2347EE7B3133CC275E601AA1874BD6974894C99D9659C1910030AD033138A57F8D9B312208873F733DB9C73007F45C49A007BFAB03AAAC3F107C8C0A032CBE24602A2BCAE4D2658D21899EA89EC5E832C0565701E5BAF0DA89E1B55ECAB7802EC830B8268F6D9CA3883C97AACC20FC4861EB59707AFA18A7B24534956301F238DFFB68BD20A8264ACB6BE4918C85282F63CBACA87A3530515041C5A00CADAC1A0B365B8A6CF626B30A49338461E94B44E20C250A23C7473952A20404FC17055E124792621D417CC7E20381BB24A701D428B1643BC0054B5FF46BA9129C0126889156BC6418A4C3B8C6D41B3D8646A915592305C30886B3995400A2A80771A494795D4B4B9EA6C21E0B21E0933F9F7C5FCB3B4C20AA303F655CB003526F8B04B3C85260C00C8E316DB495417DA22BD3A975FE784434C04DFFA545F91555F01033696BC4B87ABC0665B7A5C972D9A08FC8911147DA3CEF156B5CE85391161F32D09F6F8C0F8A845E268AA044E150944247E2B4354CDC72D3EA01B5970A6061B4F83563390BB33CF1CF9F910858DCCDD6C65E463607DF5289A3684440546816D4C2D59B214A15347EA9580310786A182EC6C2ABB73062FA281AC6806A6A042C0E95312027AD66EA6E245A9F2681A24CF505D5DB43F8A8C39BDC12E07C9F805634C9E5725E18A4229A20A681A0CA5144A8611DC372BFDDF102E5254AC1955450B94CE601AD68B5156996AC8C8A1BF1962A2758B4CBAAAB3A341E9C7C19005AB82BD236CB6294AEF22F1A1BBB32DC6EBD1C15FEB409E6C495EF1013E979981DB92BFF97BBB3B34DC385176C440B29637F68540BD10143AC33CDE4D2331AAB1DA5D9C7A41344A073B41FE536D5890239A96A319852593671F6A22E9480775F265C66D68521A73E9EC83DC6AC2963D3B70C3453861766048B18A135374ED53521A458E1FB024100116F434D1A261CA1C579F4311549243AD629525EBC465F8779C7E1B50344009C487D84A7B2683298EC39C27AF80FB3F28C61D388959A99721857CFD30F98F43A19716CD9E698EC946E69FAC477CBCB8E060EB9C96D3EE64B894C748133A9BDB5758CE45DDF791A68D87D1EF70D9AF626C3A909451A132C0702F0731D5D5959A732684C1759D8D31092669E70ECAB9F804CEF4CA416563AF1F27A219B3FD6BC482901201ADA9836B824AF9BB870DB2C9CCDAC525933E58A16C00B373DD81A3B89C39E9893960D7F6CC246A53DFFB9FE", + "dk": "09766134E63C40A74D70107A121B44651CA43F837316F63F01290286304DA07B45FBCC418092B9FEC00EA7C88A4F20A0ADF9188E23707C88CFFDD6CD86F35A9E9A3D4F842C84A5C8662048395099037ACE5FBA75495A6370B9118D11324788BD4A841C90D219AA6BBC5C829EF2C291AF3AC99CF08672D95225DCC4AF7AAA428B7EE7201EEF8966A10B88067988F62C27FB52470C036535D762DF868BDAD167A383C74246B218CA68EF9B5106979E30BA4B278A731EC370750C2B37135D39B32F811AC667E717BE098B3F278F93815B2270C01723C322F1A3F4D8B1026A39808A9DE51B8453507464E283694BB937778161E604E127CA85F10CEA05CD28458503F100CAA530874BA500A3102CAA3802291DBCF72D457637C557C8F70012EB78B9DF1948F6055974C81AF1FB6DEE601FE40C3D185251441437582B57AB29627BCA1804C3B53EA76325B925982C8CEBF50474A5BD00E43274437229D94494B7536BF473435A1F3C552C45064F4DF31CAE2338394BAFFC8354D1005CDF0CC7D5A8782906135739079DDC727B55A67515C609223FA3C34B2EC24D033397B0D8C6777547E5C635F5E546DF52A207D2CE1AC3A9F85179A8898050078E93093B24122A7F553CEC39BF42B35A07F128288A23C91C9232B835A31288461640CFAB5EAF9A9BAA63AD40044160369C899737FFC8590C83A20CAA997FDA5BE4367DD00B91131C0F4EF61B0D207EEC750A8F4ABD5CACA994B779183051C6F52DCA8696EDC65807758ABABB04203234113052C7F894A478B564EA69D14A1970D2BBB437908F2288D0084EE7D538A3FC628E311D5D558CC2113DD1762DC3FCB336A899B583CEB457A8D55935F6926E1B5A2968352D6B1A35C986915C248E8C193EA3816300C67173930560E8455C560F3817C58848CC9FB3888037B449AA9268876AE4F665B5838329387431408068125F7DA65027917AFD42BD01389FE1F64BEF5B0EC688396CE58D622975D69818888029F6C0005082B9AA14B0EBF9188CE93C3B127237597FEFCC4DCF15B7F70518D8A9B34F1826C5B01CA9F48D1122AD41139C2E625F37DBB431957CD4DB6818292436846745A933014200EFB80FA9B4A4D38824CDD05C1422243A440126855DA2948024E136A23732B98A3319809948B9AE25ABC9B6A5A96AEC8A50167DF81C475EE586DC137DD5ABB7D91A0211258A82982DE2114A808A4DA29B79483A26F3CB73E5985BD68010F233AD7C0464C39CB0EF02938F036E1B099313318E81089CE78C1D008496B69764C08C7BA7DC00328846268B0709104AEBB8C757F22A8A45274CF75E80B1B377A971940852DBE1579B59B7B7901E06ABB7972B60C9357CCC0024C14341EEE51502D79D51D87A6F42897F5955A2B44C73766662F940D47B7DCEE02E22B21E138014D735404A13523EC23ADEF21A3184939AEB0BDA5264798146E1702F42C58185C3B063E1B884455C2B508DE26C5D334C0020C7430D774AAA85B2F9007FA41495A50BAD7315C4CAD429BF2B5B733690A8E422506C36FBC877708BB9F51CB899A48D3885B73933972F97A31DF12C37E2CCFF5608FE500F0B44585CFA0DFFD6932A2096CFE13518FA06B2E93C6F31CFC6E5C1973C2FFEAB73C0D5AA0F7A2C1B922A62065A423187B30B1DD261104933A036A89BF1357961008707D7C4819C6746E137531797E2B554E4C06F57A1811F9B1C0B459B2CBBC4C90293821AC6627B20A1BB8F93752222BA795E0448735840DE552AE26C2B8BBCAFEFF61E0B7898EF606DF88156E9540CCDDB6F4ED180E8E95D2404CD7FC15DACDC4873A2276DCC1892A36B4EB8C973D6858A478FA669052A04CA890C4FEED830553670E5F07FC43C59A0584802E541C26AC4C3C5774F79CF2251274C54BDBFA5B726CA710530C87EE49918A41BC50A3CE5A67C05689646516F2E77A832622EC11C9A69F70B4D014FF35182FD8B6CF2E181710C2EFDA85358712962AA2F04C58360756469AC4BFE861EA019923B68A4B63975064596F39B83FA7083620C20F578B7C96CCCF4FB5D485726DECCBDFBBC6EEEE2185FB46379575B5A875B28653A6945A7409C0402EA40A386A4F0F47D5BB1C972E96216384F9C48489CBB9DEFA92B2B983E11B796F0F176C6D5264290AC88DB13AEB93C022BC8C2266E88C29C2A91CE80B13EEBE182AA709A1694A456B139DEB1977D531AD2722AC7D2B356AA2684D767C60012B1A2345A69AC52C7A814858E8C741A80C4B5E942CEEE0045B957748E2847EA835E1FE17C70456DBEA13FCEE0384BF431225C8A6B499624F086BF71518C8C269640B436E7783057372825768670B2478C96C4A1A2ADCB1F83A057263362476197C5D878D76B025F83AC802CCC02296A99296CF6B23C5617AA95387C2B0071C23AB7BF614371BC7001E29D7109057DC03053FB38C0084C48890612369C649C0BA3C2603E377C84D15B6DB6BB6E9513ADA9A47210861AA055D14683F0556729192CAEA8AE9A538FF810596D64BC0964B380CBC2F5D706A0A8135C1B87C3480C6D54CFC28A53C339666E24213D33A62EC872D803103987A36256840DD98DA8BC17ED42C34F5C008F6C678C748F3F7404D17B46F29012F079A46A0ABF77C04538254ED4B6BDEA929724AA60014674C427C55B1B0192CB284ED02194F9615417B08A9580DCA78944F43094E48AEBBC8BF0126310D9AA79530457E04DBA9B0D02B045AB28391F923C25CAA69AD038B4586350613D9AE22896F2979D56BBAFDAA294041AA42C21B9AC59E41CBEDBC33AD6996C3451B7DF5B9AE2D003F5092B91847290336A62E804B5F167D7CC4E31F40F6C640C10EB99D7F5519C80147B77843CD083B3B5C03718B1A7EA2DBE25ADBD56762E9150B46B6B4E82B769C384CAB39B3610070C6B6E5981C3B182105981A50567514CA15EB44620C231740FEB6BCBF4980F7B1824A5602A982A68BC8961BBC441B111C93B2374EC88FEF30D4931AB0CE3B5BEA680B1099CFA79B3FFF2B56BB3031A8C07519C7DF4F75A8C887C6A1B7249A700717B2798C02F0EA04E720752D8323376E1635331B07435CDD5BA6D7228281612BF060C694BAB6A00A8700977518BBAC8195853FDD34D44E29FEA7286DCFABDF47C9B2F2347EE7B3133CC275E601AA1874BD6974894C99D9659C1910030AD033138A57F8D9B312208873F733DB9C73007F45C49A007BFAB03AAAC3F107C8C0A032CBE24602A2BCAE4D2658D21899EA89EC5E832C0565701E5BAF0DA89E1B55ECAB7802EC830B8268F6D9CA3883C97AACC20FC4861EB59707AFA18A7B24534956301F238DFFB68BD20A8264ACB6BE4918C85282F63CBACA87A3530515041C5A00CADAC1A0B365B8A6CF626B30A49338461E94B44E20C250A23C7473952A20404FC17055E124792621D417CC7E20381BB24A701D428B1643BC0054B5FF46BA9129C0126889156BC6418A4C3B8C6D41B3D8646A915592305C30886B3995400A2A80771A494795D4B4B9EA6C21E0B21E0933F9F7C5FCB3B4C20AA303F655CB003526F8B04B3C85260C00C8E316DB495417DA22BD3A975FE784434C04DFFA545F91555F01033696BC4B87ABC0665B7A5C972D9A08FC8911147DA3CEF156B5CE85391161F32D09F6F8C0F8A845E268AA044E150944247E2B4354CDC72D3EA01B5970A6061B4F83563390BB33CF1CF9F910858DCCDD6C65E463607DF5289A3684440546816D4C2D59B214A15347EA9580310786A182EC6C2ABB73062FA281AC6806A6A042C0E95312027AD66EA6E245A9F2681A24CF505D5DB43F8A8C39BDC12E07C9F805634C9E5725E18A4229A20A681A0CA5144A8611DC372BFDDF102E5254AC1955450B94CE601AD68B5156996AC8C8A1BF1962A2758B4CBAAAB3A341E9C7C19005AB82BD236CB6294AEF22F1A1BBB32DC6EBD1C15FEB409E6C495EF1013E979981DB92BFF97BBB3B34DC385176C440B29637F68540BD10143AC33CDE4D2331AAB1DA5D9C7A41344A073B41FE536D5890239A96A319852593671F6A22E9480775F265C66D68521A73E9EC83DC6AC2963D3B70C3453861766048B18A135374ED53521A458E1FB024100116F434D1A261CA1C579F4311549243AD629525EBC465F8779C7E1B50344009C487D84A7B2683298EC39C27AF80FB3F28C61D388959A99721857CFD30F98F43A19716CD9E698EC946E69FAC477CBCB8E060EB9C96D3EE64B894C748133A9BDB5758CE45DDF791A68D87D1EF70D9AF626C3A909451A132C0702F0731D5D5959A732684C1759D8D31092669E70ECAB9F804CEF4CA416563AF1F27A219B3FD6BC482901201ADA9836B824AF9BB870DB2C9CCDAC525933E58A16C00B373DD81A3B89C39E9893960D7F6CC246A53DFFB9FE7708C928659DFE465AE813012DF3FF6EECB1E081391B0A94172AFF81AA3296C4CC96DE9E1F0506BF490A154A6DB46BD2BD1413C3D1DC163190FF171EF0D11E22" + }, + { + "tcId": 53, + "d": "F2A1E83CAD0055A85EB52F1BC1B9EE24F7F58D9C2FD099B5D38C9F75AD657578", + "z": "26D7035CD963C74AB142936A91EABD3074C583654945F9F5FDC29F754B06EBF2", + "ek": "9331AFCAFB47F0E805E29B98A2A3C2D9D49AFEA806FB1CCD5E24105FCA6C3CCC1551B613927AB4CEE284CB53968DB3767C2154295CC479F06A4560385BB16B29273386320FABC520E5B36FE4D85814079E7E835D7935172DC893FF9277B177B7D8A9ABF9C66DC6A52C25852639540220FBB4E2E576440351F281712910CBF6A02F5A1B786B006DCE399794191D9F6BAFCEF1545E3A960CD204868463FF2B8A20944211408CA82AC182D05AB4C32D6A556FC36381BF163F69377E0028C8E8C70F0AD7A526CA274F9B65BD3ABA9164BF34F74B47C56966E4BF8E1B889E54CA29A037E67C28FE2A2B6710554D3813B55131B2B75DE0A419AB8888151876CD7B476CCA91FD83480D150CB3206F6A40B95E3412F9E7288EF051D3C99ED40B9E29A0A65CA1739F88122C72CB15950EB99302CCF18EDD17446F165930396F85E84E52E221B5668431B55CFA08D048A1386D450938D48CC2A94B38EC1F3F6AB2FA5565BF3B4BF1638AEB6A2440E5CD840C564183331E1AB7CB114A409CAB6D88C84C4545396360B1FB37B36CC405451D7FBA970FA7433B879EA8CA0BFF7C4BB0B98B255412AF695511A9CE168BC4FD50C27E121CA0221DE918778B941019D32020EC2F306C7BDAB38F9518BD51FC374F84B5E4B700F05785AC39CF9CBC0DA4C52150C9485D4A6BB4CC3C54A21DF63C2044B97C8709A5DAA82933197F1EA24A994370A2C9495E0B6660A10AA62BAB0BDC4DA284CC39B7C0E0464798A7365EA106793815B7B2BF02EA70C5B506CA343C38386BEFB69D6E31A1678BBD497BB1E64075B4158205280802234AE612A588CC8B2EC24AF74C5EAC085EEEDB13B827340AE29F348A9E31A20244D20A4B2B41F46A6A4C8293A2B8C5E3D3C49CA1AC0724969360BDD8A0822DF35D16A7CA48F90BB2C7972CD70B8DFB6CF5F1453F69266B04C2C32987DE0565BDA632F4B257CAC7CE7953A70B29681BC4602F3ACC3CFB3BFB63A135A758E461647E344A454AB3A1A72D234268D8870ADB84AAC250127E2AA593E79828848C112C26592A89C4DA358BF20571144C3C715449B8AE46606A51E150F4CA3593BC9ECC8A0382532610328253F488E733965C1742A36626524809C548397A922E79B82FC35B1D0EE41975CBCFF6F38BAAE28F0B452BBF792CD9023EB893767FD36E855774F1600399DA75889B03A95A21F32C0162540CBAB08B82B8346CA43B3E07B44F8C5AE6255CC379B3FE631A434349533B85C164B44D7805DE49442D113D238A9856E1897F0352130369387C3F2BE95435885A117072EC762656BCAB6EC1BE33C80C7835C4670B1D142767E23BC0AE6247529A3C8F348BA37ABE3507AC93C939560A445CE696881248F2A6C0B19B3900DB2E803A4BE0D4231F41A65702502241BBC724AFA441991A0A46DE7C62F6A220E457B2D876C44BB0A5F3116868E2B6D63A44A5C55F53F83E10E534606B76108CA4BF12A1B97A61A28B1BD184A9C2057655D460B65C407CB93272F44509C432DAD7B03A7393CFC160861CAAD4E9C81B9CB1085539D8A2C1855421346ACA46F46C3D12696237B237398B4015467CE7A6030A9666736717E6A740152BC2939998724018525D74724C5BD35B2D1887F9D2C6733A1173129DF3AB625E48C6664207C4D4931AF4130CC1CA0EF9160898890B49674F92A9464837F5597A7CA26BA854A4E0FB756572766DE2B9FAB3820818CBC600AEEF588B59B416F9568E19FC4B03902CC13547DC18197CB1AB23190424EC80633A5D58D861D9BCA5450102E52247A74594936153723C0A561B64F4E75D90E805DD7087FFD3420B4A0CC54CBB32944693A393511CA69E2C2B7A3A5B58784F2E999840BBA8C2C994852151569CA0949616DAF73328B220DBEB7D43794B1373CA3E8AC69178653E889D67E8546D35A40478B9C6410D9376CB28D6080AD41FE834B970403B8687C29011C6C8E24137F22FB7192443190BC2B095594C85C55389D4D49722124F07067FD9F34319565392799C27550934127538524E1655A03F697313130CB0F6799DE565A5DB8B7C23C7A8F53AA7714C7094A86E67894A75A489E6C414B67C52030F40EC5C599969D8E00B7C03492C8CA6DFBA8D641307921712C2840591FA9A847B39794BC22747830FD956C0C17B5C974FDCE96F565355B500A70C3D86EA32C9290514B1373184AAEDB75D90976FE6765F4747BFC946F1", + "dk": "8EC50C8143608F9186988602849A965D48645CA331C1815E506A1C51D73AAF18BF92D39BB18A5E681447EB78883CE48E6C375AA8E36CA82ABA3E188D8D795DDEA50B655B2E9409C35931C934BCB5820C6D2CE975FD955A9BF40695650004B4C62447AF3E1082DEF45A4AEC121DF9C8D36C8264034E55F926BD5053421C809041C1701650EC0959E4EA6D09898B8BCB90F95209C3397525D984F6079063901BB82478F9B1CE08875449355FB16A3921A8CCC9547349C5650BF49D3D9BB523092607F2B4BB437B1A969C35525F321333E9436B66A08699A9CE4017C6D7485145C408FD412E69463A558927F0763ECD8B24C344640535CE10DC41780213A3DB5930DC40CBA012CA5B22B2052C3192B31C9230F4D12FD5767EC1700775C478E263220E40C65EEB36FAA1C18B530B6C5ABF6B11397E694763C8C781CBB3C6C767F24287BCA224370C9D5E9066609078BB4B7BAFA46328F9C203D7AB0419B9177295DA08B802D93A97DB3F1FE964BA7B78F4A5404657603E5A7B95971A1DC2A1B00B54E7FB1C180CCE49917278C961FB61009200070ED90BB752A55661B011E35382A01C57793AFD864AD8D3ABCD752F78168EF12238CC520A6845CE3CA5771D207B3A2A594E687867783E80B057423118753BBCD3926B6640321938A40E793BADA8476D64858296348BD7038DA24E65D4B6FC4C907ED69E0159C76AB07F9111035B6187A41024AB26966F7B64673860EC9AAFEF29C0D7DC8A4DE31344C41D84D07B3A6525A5A884BCA8870E758E4A6C02150505A7587EB6288C80910C0E999C0309A34DC7C47053AA1F01848BC15059E65307207F4C4974DE0ACE30C2A3021B5927B1B0787002AD0A4CABCCC2D30865DB9150F657025658C3A14CC7D088BB1247CB2097A521FC46E75B158C55509D0A6DC8FA6B53016F6C28939C889943075E1B26646E3BAE5B0A52D3553E3D24AD62396AD18CB53C896E8FFC148F57CE11A39677052C134B8D0BA186B6F9BB94A2C96D06627ABC7E29660B9DF9C9B9E1786B39B9148A982B456F4F426368845D0377C043036252A4BA1B61487958A9E05CB272072CBD0927B4667BA7E4837E4B7405B9CD3EE43F7575B8209BCF122622E7E99B414C3AA32C9DEF1B200444B82BCBBA8EBC079FB42E00A830D921C45387BE4F86BC05D64BCE8B948BC61D6FB0CBC9552591F99622C50C498376A106414A6109354184EDAB57D8BC27F6761E18E12A12335BE89944B737C3B3BCC3227148A5A60EA32B83C8C5692B35BD68D72DF7CC264B519C52437B357899B3409239C02251F83460F6AF9B7C73AED1B4EDEB322F794555A0CF5AFC233260A10C6529B5AC55582B153461C25401027748B39260015E00184D553F45372613631E2ED0B4633A0CC52B6F62454DA414A49B17BC88225EC9EAAFBD121494652C925BCE19CB53E3216040B879F409049B780098A189668A7C46751C714B38A6E59EE4D7620AA3B3DA3BB3FF3C4C04E45F3ABB3CEC417CD80908EA701D90A21E22210CB7188D739CB13DFA03E35327C1912453C5709C1B6FEE2CBC5F8714E1A43AEA7422D6E63F247C8065A0CDFB782BDD2CAA781B4B8B76BB7885C7B446BB40A52BB2C8112F3C5DDD077FAC625D54DC26B0A3B52109C8C80137302738B511377376A74E07C1D2C3108180C6ACE97E61F6A343D6C1CFB76D5B18B756179A66D51483B904E5B88FFD700971A37E913829A2F366B8386BC342BB49411890C958040B7E66CCC5FF9C55CBC85209784282F865C7132FD0F82A4B016A9CBB64B03CCDA3F64FE6E760DFA764A7D2A17027B18C8909FDE5C9686608670871CD95B2B1DCCD99C34B22034147F1471BB38D3043ACA4C727BD971C4676BB7DBB987E015084F16E05686994A776704A161C7407007623964359E0F77214B03898C51B4EA6474A7B0711E505F0060C220A80A9C591481643A9796C2AD651ED7783F38633C308B33490867DF28DFD054AE9C49CFF78B508234E17538EF58172E92642F43A5D421B2929210BBEA122853918EFBB91A0446405519D43DA86031A6DBB3A9E24BCBE4E686EE9EAAD71A42A4D27907E89326FB811A5663D18002F7294CED499A29EBA328F5460F80512D42AB48B726E798778A2C2692BCBA2A0052F9A56C1A0631741A6BC846A8C0BC309B7E9709331AFCAFB47F0E805E29B98A2A3C2D9D49AFEA806FB1CCD5E24105FCA6C3CCC1551B613927AB4CEE284CB53968DB3767C2154295CC479F06A4560385BB16B29273386320FABC520E5B36FE4D85814079E7E835D7935172DC893FF9277B177B7D8A9ABF9C66DC6A52C25852639540220FBB4E2E576440351F281712910CBF6A02F5A1B786B006DCE399794191D9F6BAFCEF1545E3A960CD204868463FF2B8A20944211408CA82AC182D05AB4C32D6A556FC36381BF163F69377E0028C8E8C70F0AD7A526CA274F9B65BD3ABA9164BF34F74B47C56966E4BF8E1B889E54CA29A037E67C28FE2A2B6710554D3813B55131B2B75DE0A419AB8888151876CD7B476CCA91FD83480D150CB3206F6A40B95E3412F9E7288EF051D3C99ED40B9E29A0A65CA1739F88122C72CB15950EB99302CCF18EDD17446F165930396F85E84E52E221B5668431B55CFA08D048A1386D450938D48CC2A94B38EC1F3F6AB2FA5565BF3B4BF1638AEB6A2440E5CD840C564183331E1AB7CB114A409CAB6D88C84C4545396360B1FB37B36CC405451D7FBA970FA7433B879EA8CA0BFF7C4BB0B98B255412AF695511A9CE168BC4FD50C27E121CA0221DE918778B941019D32020EC2F306C7BDAB38F9518BD51FC374F84B5E4B700F05785AC39CF9CBC0DA4C52150C9485D4A6BB4CC3C54A21DF63C2044B97C8709A5DAA82933197F1EA24A994370A2C9495E0B6660A10AA62BAB0BDC4DA284CC39B7C0E0464798A7365EA106793815B7B2BF02EA70C5B506CA343C38386BEFB69D6E31A1678BBD497BB1E64075B4158205280802234AE612A588CC8B2EC24AF74C5EAC085EEEDB13B827340AE29F348A9E31A20244D20A4B2B41F46A6A4C8293A2B8C5E3D3C49CA1AC0724969360BDD8A0822DF35D16A7CA48F90BB2C7972CD70B8DFB6CF5F1453F69266B04C2C32987DE0565BDA632F4B257CAC7CE7953A70B29681BC4602F3ACC3CFB3BFB63A135A758E461647E344A454AB3A1A72D234268D8870ADB84AAC250127E2AA593E79828848C112C26592A89C4DA358BF20571144C3C715449B8AE46606A51E150F4CA3593BC9ECC8A0382532610328253F488E733965C1742A36626524809C548397A922E79B82FC35B1D0EE41975CBCFF6F38BAAE28F0B452BBF792CD9023EB893767FD36E855774F1600399DA75889B03A95A21F32C0162540CBAB08B82B8346CA43B3E07B44F8C5AE6255CC379B3FE631A434349533B85C164B44D7805DE49442D113D238A9856E1897F0352130369387C3F2BE95435885A117072EC762656BCAB6EC1BE33C80C7835C4670B1D142767E23BC0AE6247529A3C8F348BA37ABE3507AC93C939560A445CE696881248F2A6C0B19B3900DB2E803A4BE0D4231F41A65702502241BBC724AFA441991A0A46DE7C62F6A220E457B2D876C44BB0A5F3116868E2B6D63A44A5C55F53F83E10E534606B76108CA4BF12A1B97A61A28B1BD184A9C2057655D460B65C407CB93272F44509C432DAD7B03A7393CFC160861CAAD4E9C81B9CB1085539D8A2C1855421346ACA46F46C3D12696237B237398B4015467CE7A6030A9666736717E6A740152BC2939998724018525D74724C5BD35B2D1887F9D2C6733A1173129DF3AB625E48C6664207C4D4931AF4130CC1CA0EF9160898890B49674F92A9464837F5597A7CA26BA854A4E0FB756572766DE2B9FAB3820818CBC600AEEF588B59B416F9568E19FC4B03902CC13547DC18197CB1AB23190424EC80633A5D58D861D9BCA5450102E52247A74594936153723C0A561B64F4E75D90E805DD7087FFD3420B4A0CC54CBB32944693A393511CA69E2C2B7A3A5B58784F2E999840BBA8C2C994852151569CA0949616DAF73328B220DBEB7D43794B1373CA3E8AC69178653E889D67E8546D35A40478B9C6410D9376CB28D6080AD41FE834B970403B8687C29011C6C8E24137F22FB7192443190BC2B095594C85C55389D4D49722124F07067FD9F34319565392799C27550934127538524E1655A03F697313130CB0F6799DE565A5DB8B7C23C7A8F53AA7714C7094A86E67894A75A489E6C414B67C52030F40EC5C599969D8E00B7C03492C8CA6DFBA8D641307921712C2840591FA9A847B39794BC22747830FD956C0C17B5C974FDCE96F565355B500A70C3D86EA32C9290514B1373184AAEDB75D90976FE6765F4747BFC946F1249DBFD72887FAACDB1B900CFCB5B3955EF374D86072D38CE29DD9E906AFD6E326D7035CD963C74AB142936A91EABD3074C583654945F9F5FDC29F754B06EBF2" + } + ], + "encapsulation": [ + { + "tcId": 51, + "ek": "2191ABB6D6BEEE29C5780758A970349879B61A028DEEA5404731292346C81EEB1D17766AFBCAA68C867D91132F34A494E28CAA767241B50902F4825771865FC8D633736248963A253DD52C1A07C7177CB6DF74C43C3C74D7133DA9DC915FE14B5BC30B86153E7B86BE04189F4CE8CEB6E5CB69808D35640AC50335B1633162E28260300646153BE7B26A84C1C565956F8577C95CFA80B68978CD7A79E49C7E808B21CDB460ED8388C4859670333B69980B2F9A0F78D7A5B9D8554893A0B5C52447B15318828989816F43406ABFA4950607A5A57CBA62F5C4340AC3D294191687C7D2B60EF2D83F6BEC15B1618129B78577447BD1E36A87142B9C552C8CD2304F35B81B3BC5196C3840C0346D513EC8862BA14A172C051402F7398707A6133C8E88E5B57F9B9EEC18AED1404EA1C47BE5169E5D558E262A231A75925D25C499650C8AC031BE580CFB185A0E0362CE2194C53713A630CB59600C1942908FF5142B02364B74B3F5474301AB2DA4188DCA640961142F28584EBE810683C046B59CC2CC07025DA63A5E8C5E99E24FBAE183329886279C2D58D0B8CAC1C7B3033F05E765C75414748687FF03C00B36709D41C5968528F178CD03B813B69054D3E20FB6A04A713B1B169AB5F0369810FC5CD967A866B70A2D87514605794930BE9B7B5D49D08F5CB08D84FC5BEECB994DD2A5FC10AEEB4CCE1ED251FD4815A9D06EEA6A77F18C56ACABB1F7C478DD52218B7BBF9770A51C3AC96402548A1BB22DD0C29F422330057179EC2D91E1AEE27BBCB1EB4671324265E28676090896E332535BA2E9169C54113F19A45229C2448078AE08F7213F347244527EF726855CA054AB7A1ED1A40A285A60D04C00CCFA8FD318CF182705DDB888DF652669B3224CF54ED6D00DB5121331B40C7E35B1945925E5AC1503A999899317A2370020D81873330327DB7CC1D430E32C6570FCCFFEEA90A871A4B04CA9A8065AB20A6459F1806221A5482802F14A68893B111607C4E085A5A6410C027C1BF5FA33ADECCEA576211F272CC942ACACC879DE398477780A26E7545ED693EDD66AD24772B0C0C4B033A73ED94E1D02C384D1A288A78F5D8838B5AC0B37855DC59B987EC9B0F5C6868B0319F3372024FC15B247144F925D3EDC453FB8551EC1CA734A37A3972B8CC9B84A217A4D0BC0D08C815359ABA59A07EED998133587E062CF735A8521A18AC2059E3F142F12CCC5AE024CA33B8E4A285BE5A3CED460C2C3675B956294EF2C61C96A88441665D56B2029688EFE4540DC35ABF5957BF9477F58277851719547510F22D01A13D0047C5B88372A654C732AFF395DBF825829A06E6AE6923296218AB9AD33D0B07A0B6A3F3B3DD8428A1F82B72A921853D94C5D400494494F0D168148B5392F386B16C6056D6B8D767B7F4D918A9384C855C8B4DE8450EB6CCC2DF18528CBA0A3552D5704683AA3AAF52751FD5A17801A76A58A75038533C1576E8668B5D290CEB20CB1FF91317D9B418311C11E617D10B4B0E9055AEF5CC6EBF8B91F0244ED9854FDE463E55112A238038750CD016735613B9C461996B90A3B4D4B0D8B8825EC087FB138AB868A04CB3C9249B5B118F81ED06B99E6B521F206A32A4013E876C6C80B36AF39AC197A21B78B5F2713AA45C74335F32676C2AC2902710EC42AF5984687A672E0466831C7B436300B30351795B22B3A939347A7B744831BAFE78C3BF09A8224A958568F75D71F822941DFD64D8969ADA5B73C847272B5914491BBCE5F0C09EF501BABD95190F93201AB413E833F7924B696D5C2B62272245932A6D9C35E1670AA3BC57CD41B8D61A52AAA6D80BB774F922ECB41CD58469C4DCAC698F6730890237602064B148650C17E6EDCB3713382FE755E46F3ABE0C6C45184AE5C5C861E35A0E2BAB2B4E8A4775584351170152512C7E99E96F285B916605B53B91D6C501F51A93129483046CB2DFA16D4A540ED2B6AB18BBDFBA2B201BAA20297AB1FE7133757C07DC041971AA4219157D77628F31AAF520650A2250ADAD1524E58268D21CC62E2469C8425E86359DBCAA066ACB466E06947A1C144E53EE0DC930432C806C493B2BB25757B1E2C775A27A896EC25B58E8A78D4DAC91815A1F3D90635EA25AE13CEFA77877DC0C2F0678ADE307243B438C54AA69DDA539B8A41C1B3C98C77FCEB2C0BC2025A272D927943EA338CDF32A0DEC8B187DF5C", + "m": "2F1E2CA7BD72AF847CAC38CDEFC4D345909D7517543EDF32E2FC491BA05EB5C3", + "c": "D892E0948544D020ECE56F00C6495B9BD1C469AC0F134002C864D10CF61C6C7887890276A7546310EA077741D83428F22B60E2A40ED8BBF5D9227893BB0B7417DF4380323426EBC9744EF35A1BB6DAB1181FF0B677E8C9B6574360994F96EB87C3524E15E468283169D90D8A994BED0DA9CD778CA239CA6C225390221FD408A3EAE541A032D714F3D078DD1EC722CA83FFFC92A416F46FC1E8710C3F8E9CD452C016F483985A5C1D951FE2B03C4AC2C9A0BA71F6FFA4B29EDE76DF69CAD594D10618B94E783FDDAF11CC801ED5036FEB4A70D071079944B7D9F1FFBB98F123DC34A52A40DE57E079A1F8B180DA9E6ECAA47A111F3C054DD9563D7E74B95AC65AB453AFE0BF0DCEC5578FE6FBC2D9EB91932799EA64B2DFA95B2C9C6E931FFB0A5A499DDC42C6D6B563D5A6C1F34DBCB36F57E6C10C69FEADC70A3D01F5F3715F6BEA0D3E2E7AC2B04E01CA80A2E187BB7D9EFA5EE257B9C3CAEA3947D785CA56A76D0B6692A69418935DFC36FD9B26758F5E1043D5C2A6D5CC1556AED592BBCD652F4024BF324910F8CAA2415171F3EC8B1CC494F2B519CBD2B13D317672DE60F0A3D400E6079F85C3CDAD0DA5A17F2EFACCEC03D3C2E85B063AC3CB29F2FC4850705B8D472E35DC12D06A7D021DB302CA883DC7D4A57B7A9E1D023960D4EA5E39C9D5BF328B8E4AC0AFC3A6C990F598F373059AD28EDBEB90412472C637877EA82E7105CAF467A15CE961695F2FF04C750E277E78C1CE9E490A5C45168936BCF79261B95FB20AFCCDCE790E0AEDDE204DA4F6734700FC1CE05239F4EAD0A496FD8F733049EF9B6ECC54D5C635B56D6FCAC7ABA1BA32D707D888B820CE3E794C2562798210D726C6C5DB3D22A214F3F2A1F2477FCB77377AF41D0F3F0AC635E7BF0674CEBB672EA93B9504EFA8C5AAFB7458E41A1D85C28841877FF2F444005CD1C1E4227573E8151184786489C7CC39549F8A29B9C4D68BE38F24081FE239C5D0E14AD5E361720836A72A99CE88E47BF3E6A1A9EB57CDC3724CB4789FB88C135CA9CCD94CCC9BD73EE8A796BFEA36E894ED27B0791E8AD23FCDA5AE7C167D5774AA00468D97BD0FD2B9BD78D01E1E39A66994A369212D95D7886BE00F2B7A1F187C85B873F33CB7D3E4C23993F7BC9FF7ABBC6FD9D5C54F06E5F31B79DA992BC0F2AA79A36BD00483CA0ADD2DCB2BE36C27DC913668F030128E82281E5E66E87C75DCE3FC60431CE6A19C6C45E19A2D84430E945034B7DA6C506EF009493AF22F701088E01C20ACB21972FF28D284D2A57A69D108161938F028011BA985E2B925532B52EED878646579183EB0EF3FB57BE1E05C1BD2B68D0E4A9CE104E0ECDFB30268E6C007812FD88498D97795447E7F88CE8A3BCD161B1E34DCF77413B778F88A3AD66F247F7664D0F6AF31EC1DADC00FEC09D6AC63FC8A802C3752484B5B43555381043FDD30EF5D7DCF4F5B68EB89F08A0FE5ECB65D6D9418CA5283CA398D3129CD3B44EDF5E3568C953CE2B66A28B473CAB0C43918A05C5A4358D1392604495D06B395187CAA6B36050B436B218485466CB6643EDA7AC66AAEEA758715A22A8849066879CC966DA7E0F7E843A1E234920AECF2A4FED2F78C69A6C103A5A535D77976B1E40BB0D75D6E370212017734F2B0BE9F3F87C5583634EA6A998D8FE9CE5E5905F2A6ABB9D635425D06731C1227EB634A3603D081CB1A7C2B0C1685942EFD6F65992DD39AA67CD954FA0310A0F5866AC6121E27E349D5C2ADF37A672C1DFB021A855000F0C0C29489926B997930B20DF641120EF8605CFD9482EEA9344918AEC689F580A94508F318E63EADC3EB7486B9FCD7A92BA16F5E02CF78EF73F528FAE3A43D451C58261A82B3735E09D4F4679CE112803505006CBAE6EAF641A6F66B0C51EE90095735D87AD88C3D19CD7888248FCC3872939605F205976EF6C6AA52F8316E9DEF841594373FCD177F0E24D8975653A29F738FA8F0D457F2D72B7C00A4B7A66FB080F705A8BD599314354E65842AF598B8A2A40F6CB3208762AC6CD467D06A2E987A7AF72E355BEA297DA87BC9250EBE8D8F85CF292200A21D93475BA46BE78C17C1605C10B97659859F08F980114955E68361E180C98015AA46A776070C4F2B328A903A70AD226742F0022279179FD2530390F190E3008F7202C83A6A15866DF848C8A150D12287451DEC8EE7F04C1C3121E4BBA687D7", + "k": "5087E3B0C90BF601DD6501E071270EFF8683621E9F5D67A7A668E50C4F460A75" + }, + { + "tcId": 52, + "ek": "7ED2911897C09E483AFFA62DA7DB18BCFA54D700745D055844778C0804B94BFABC3212B6032C6BA86CCAC5B844A0C1C1130922423A628686AF96EB08AF8BB51482A531CB648B4C535047179742388FA4CBC04118486AA340E50D01F38C4E305833D04D90BACEF4338F59B91AF2D17ECC583516813B8476BC5E4AC5CF4A1E0C621DA498B87FDB6126ECBB79D3CDC047C4EB8B07BB246F2E11BC76D033CA22A92DC041CD646154C0528EA207DAD8066CC3068A026DBA587ACD78215C6BC794C3A61689B6151C7E412148A55A7AD955B85B284C58C56BAEBCA1AA7186B09162A6035B6EA55122C67B823996D2C548C620843D6A939BB2791148CBD18145BAE3AAD82C8A7330716148567AB4AA2C93C739C64B3E63164FCB1A8CD05D9B6C3EFCD3B350F57FC98223C097AF02B77411F9AE3B4285E375923BC63E61B587C396460FC668E9369E5D1341685BC49A090C3B5C52597A7E84C1C9CA2924682808EF45B4E4459A66EC368C80BE3CE2CA5C2007FAD566E7E07C78D009D3CA006F2769476871C57B758967CF39B3166A75625914B90146A1D5A1A3A22164B1D61431E98DE06185436506F349044962A6C5D034C1BA9FAAE2B58E25AA611BCF39383CE0C43DE10B8ECA58347A9A970E56651FE4B6E163CCB9415D7A79B4760193A5CC5C9AD4251EA3AAE9231246D82C7B4ABE1D5932CBF9C439B50C3AC741F9A50FAE32C34B662F7C12937A27BA76D357CF42AECBA7C2075AC4D3050476B43DFAB4380D1854E48C91EAE54B022380156C240F6018F35A5A3933745A887B76BA9625C868C5938E6C618BC2485D20483237065C53F535C5508A8FE82C282A298F2915263059C0B8899EE2B29CB1A45D9A28DCC60179B15F58FA0CDB79338CCA8D8943B2D32B0F4B6408CFB9190EE0697F81AD4285B3E95519083C315CAC73B2920F4ED0B672A95F7FC3873DCA5337E931F79719687078D876A88B6417DA2447827987A04217FC509FCFA99FC2966DDDC05298784BCEE39390B3B48E59555844826A72709FE210350669496C66FF846B60E66EEB957AE834B85C3065340B4CF25574DBCAAF5B732F376256D7267412010B2CEC49653C7222366CA274339090A402E2200E045AE6228848219A08DA8C33403C5779B4DE89475E8466A956C6F378A4A0F79F8F22C17A516BA5CA5C3D21C80866639987A270619B5EABB34B818FC035773A9BBA1AD1179349489A49744A46AA826A7045E1ABA6833043D611679938C29C06EE896665024178DB47CF242798E30278C664C90A39335A33196259A4831F5D875085F64488B054D348AD996A94B37610AE978EFA7926F844983430A797381C0649384E299832F0953C10341E89783B70618A60AB07680D8A677B5237743B37AC4972CA72C5137E8AA4939913DB91B2556AA4A17C09FAD7064289BEF6879BAACBBED5849C93616D3A2BA5E1C0A8F037A7ACA7B89541233586788C9B29F07197B56AA26EB59C727A7502F98F65641129D310A9B036E783552548588A2C5AC789114FC04416602044432DF8469951453D58704279D47700A257944CC1CE1A2A0DF0BF6DE6630EDAB65D4995A023C4AD351B3607578B51B8DDCB43C2549952727EC97066C24230DEAA4BFA55CE0D7394B8F0B84C969F8E443E71C3C38F0B71535B1E68E7A3B08B5BDE9C9D78E655014388AF8CB54AC5B9CA41AB5E2A4BFBB58D0628B4155A102574BB4FF8C172B34BE9BA2408E0C8DF7339A009452D7023F1F912C7C84D5D1A415F1894897B7514EABB771CCAE56C677C506495595012B25AC3409153952699B9A691C09BBA597F128B7B7634004D5BAD0199C213DC892A7A9B7F51A0012C81C000BFB50B2CF7B85A94B2903365BE3B8CB4F92A6392B75866C0908F52CF402A4C8746A2ABFC91161C70B2C21BDDCA2907E29C35FC39EC610B84D28B58D178DE04388764AFAEC4C7E596C953F5A8396CBA69EB99BFC1CF7F0C1F4DC20367F1BA365192F7C35D96D089E0A5934F467EAB3483C8C0B009627150706BE172243CE0CD14684564E586A1E985702CAE6F90B8A6E2771D6059F8C9041F8AAA7F5C06CDA09428DC9904B6AA2B857EE87A3F07AC037AF379DC6AB7ADCB59F0E82E73D474E0C66036B71061EB8F111869EFF94ACA27AB15D02E84F146B9B6B0729A314C96C71A72C583FC446859B01DC664931A3AEDA27E7197BFD78421C3A5E0BC629E393F", + "m": "AE818F586EC2ADADC7AFED63C03344B9DC0C834133D202B43E2372EEED31817B", + "c": "CB327BE1BAEDEAC4CDD83977486EAE157634E3A7623260197B96E4956172FEBACEB6A4C22781095EC6DB8A91204FD3D2FF660CC52026AAC9ED73CEF79B0D39191C601A81923DC309CF5BC526FB76D8200B98438ADA64BFCD0B2105BE5663D14B3BCF0818C53BCD171FA7927E1CFE6B7FC0AFADD15D040949652B779CE05B49C937E6B7E43FAD1283AB3583E8D7A4A3EA0287EAFBFCBF10D07C8DCD6AACF44904027299F24D08C0868CAAFBEDABA6A45F5E00B1F4FFA31D8010B45D5529D3A4D1DD82EC22CC03BB75CF21FA4957EF81FCEFC2D25C53AC5C2559C57AADE716AB26ED3EF8D250C2CACD3F4945194E979C5D5DF64B097694E3052313CE86C7D36CE74FDD9BC4D3CC1FA968E5984AE93E3A4EBB1CB3309DAB6DF63FCB978140624C2240AB80DEDD5EAEA395FD0FA6A5892EA1A162923FB4372CC1BEA7B60295EAE03D200E8B2537B127ECE6EE36EE1D80DA9191029EE666CF29A35C32F9B34D126748C2AADD183BC4BF05D627E1F4B61AEDBF39F0038471A839A141AF16FA371D60B97FE47D450917EC19F5062BAF4C02D2E158ADCA6BF8302F5F18A867F9CE2F010BA079CF39A76AAECE8CD1A7129288B2BFB8562DA666084D7733BDF4C419639BEB1E8C980F86C7C884213A1325CB7BE693543BC056960AE29543D3AA3FC57C994C185642044FA3105BC8FFA80BB020CAD148EB6154A1FD9160C32BF56373084023D470ED7C003FA0A94CAA5C95CF4CDDACA3710926BC7A4F1DA5CD8D66041B1A557547DADAB39712D359228177CDFB5FA779F3446FE8931FFC78BB738A6EBB0F6560F14D6B8BC1E898F35D55D085335D92E8559E7FCB467B04A195EFC179085F5A8B400DC04CEC807E935E6F9F6D38C833DD845D1E4FEF8564BF576D75CF23D1FF74ED1E03DFB96C3B4454794D60FC88C76AD190B21F4940D0BE343434DE890CDCE2DEB58808390B709DF2B84133824A9E5C769713B2D7BEFEF842AA23A92EF9F6FDB6E0871E514AA55A80132C7A8B5ECD39441EC76D862AEA1E93EF98B9AE15CF2458DC7918926102C4C993C14B898EC430F3343647767163FB979943D1185BA420840B876D2F95C709D43C7D29F002DEDDAD4EF03B55825E19E54B1C9FC56B79E8BDCA7591F712DFF92CB9145151B3320D433A45D8B4A47651398E33AC9DA9DE758B6D80373C7DEADF937C293DF202489F99B0248F3A1EF831E6177C907F19F38758BAFCDFD7E56B5232A87E4E86735F055A164335C85DBCDB55D4B99055D00D708EB6A9067ED215C20629D532911480D2AB8600C8B474FC6AC6C3753CA7CEDE2978680032C9FB9AFF2DADF76A81AD7A7FD0639C9170EC2A78F37DB2425FEE690F40CE6D02997D538631F65F01C7F5D7A614EDFDA4117A6242D1575C65AFE5D1808A7F3FB5EDEB429BB53A0AA2584F9958F519B5FC0A4A727C8C5A92583596766DCBFDBA467351C86E3D2231BA22F656C8449CE54D726B069C790BD5C19B6147139F1E2E4B48182E609844D9D858748F7C416A6A1BA35C70A98936C2D9BB1008C390F4B09527ECA0FA719DC42016E0A8622CEBAD83F636E780EF79C2216E7DA50E5DB69BF1660A059AC6CB68AE3969A7210F9E7F8DD172080FA98B60ABCE8831C6CED8B809C85818C263597D3A1A3F012269BB3BBF0F40CAE60BB9F8725A3DAC68CBE49402C90867030F9FA9414A2BB4F5DAEEB292A00F68184150D9E7A6A38AC2094AE61BF06DB01B356C95C52BF6C0CA84DC0D9EE48802FA9EF45C07DE48D4F8E8F715497874886511C9300B950CFB9C110EF9402C70EF54C2132A932E7D5FBBC93522577592DD2F76E2E638AEF9C823A9731BDF2EC047C8B8813F086C7233F2BD8C1BC0B2A2153A9E0B406E19AA57609ABD0DF4F95806CF11EB75E440B9F4D4B2BCEBC880336C97F7104140222A9F6BE97CE3DAD5A8E5980F4C8C81069F850925CDD11799BCC5511769F7BA6434B0E3508163BBC8D9EE8829CF0AB53A7980AA0AAB5127C4A6F3F28960CCDC492BE5E79A50E44906CB4C507DF3C69F363D5AA9CEB18F8DF13667A97519A68F31D8D93B1CD188DB7CCF5BF2B2A3389DA5CD8E755E1E1BEB2649868A13DDCA3050A18BED253568B2FE7D16072D94ADD2D18BA7ACDED627ECC9FEE2A9B90098E3D4323CC807362D90D3B10AE1992D41A4CA5AFA032338BDCBF8ED165D6D7581E6A6C1BEF67C35796265BF2EB37920C2D564E5B9", + "k": "C904A4B2AC6D2D8690799AF342CFC78CA0D61E2B6BDD1D5738F32E87E07905C3" + }, + { + "tcId": 53, + "ek": "811B72ED16206C702861A17100360A1CAA903A2185AB150BC4996B4D8A72FF018545C2A933D87E412111E39CA45FDA3D870383E2829D579C26DE53916977207B851D78E35EDD29B277081AC3B244DC074DC9B7417403030A327252602298B6455E530CCB5C5E702A77492448B7F6C1129427CB28AF7F4031088778B7CB3BF78995D0DA3670057E31492F80F0457BD7A5E9719E229A82AC7AA25FF93A23043AAED482E428CEA4F33BF447B69FC2CDC4E3B142672C08B6B243E65B72335F45A929CD482EB4ACBE81561FE8864FEB5AC9C1D5604C9B19657A3240F26EBD3A3E9DBBA951FACEFC435CDC612F502C4A37DB374B353BDBE53C761BC42CD101F327AB6FC32400E307B739C2D1B37CFA8AAFB675A2EEC8B58144492F1327B0C7C8317CA0AF37882A1A8D0F340E8EBA1C82064873361BDD50BBD1B094DF7787DD2973C53C8364688C7B2CB0536942226A2625822BD88803592B1BA484A36954814EC54ADAB62E38B8013F22C101AA8B11A867B55C2BB673C225F3424263C556FB7988832A57416FFC4268796B7C5ED0A88AA9A0188C9AA51A0402B85C9B84A1A2A2071F2B962889C479533EB2630727F087B714C0223810728013BC725F8F138E2CD20B93ABB1A9561108064FEF2CBE1334831F21C66D8316B1C48B356534CDF7B1941567FC879D7639A7960B5CEC1137619577DD830F444825C89802F4F78EFEA0C4AF40194A7C082F9270A1A12E2E379810DC86E539392505A38DB4B10AF1A2242124BD5B47B3C82B3DD0595C145CE0403B53D003F354B99E872C0CA43E66DCC9F8139B006646DF49BDF225C343201AFF4C3DA8874CAAF501B1C598C97332EA6A99C1B977EDFBB99B17AEB36B79D667997FD33B5EB529175BB2294A2DC926CE0FE672FF9ACE3C5B4A565300C9185A43DB10BB08448134897AAB77606C0CCEE4A7CBE4B92A89693C0217A316CF4C1C14D334034EEC2D64797CA48683B0D8B430011D606749FAE4B8C11722B8732B88F147F3A60743043A19D97B12987BE123A5DFFAA02F8CCAEE75A0DCD49A4ED2AC5287C593C788B37C4F0CD2256406A6589595CDBA1599B6BCAE3CCFC5B7654DFB0F7DF1AF06F15BBBC973FF72BAA8A923D4622637CB70EE044B76211EE31B68EA010FEC0C922E86CCF35B86EF31A84BC560EDF91626FC64E0B24577835542186311011EFED68AF4E851B23067F1E1C236432D05A01F9CE8735095A8A00514778235436A8C383BB57EF815AEB4237C7A144DD0B35B65161A408314612BC2835621CA9AD5A5826C77974DF356B4900C5626856C204CF736B0FE47CED9355087D59D2354CD7388785F831732C54ABD1C82F126C37D87AB8C18A7F575AD2F7C302307469333729D2B4FFA3894BA4B20504CB8A2DA06E67A2102FCB65E3BBACFC7A1ED870D232B33FC0786CA64658812BE487A7C5F8660EAB206C1AAAECC6B3B0594C1CB860CAF9458FE559DA1805E038209BB2932F2D5857C833DD6F71140564749328B60C9044D8C61A993C70DA7A2DA61AACC2A23B91402EDF7448619244AB9CA807C4B527B7FAF139538748CCF9A1C95CB0DBA133334BA1DA987199E455B0BDC9C78F56827BC5439240846571B6D9A3D54A81ADF9B015B631C86C60CB5B6AB623CA63CBA8EC3B64A84F95A81F9A122475D71286C2B07C7AD395E86F44B66B377791C4D5914C0BFA160DC0B6632556E98ABC74B115731424D27E9AB12364C65D11CC059BAE38CC32007BE6052781A42B1C5B2676F619B8F9BB14084A91E06826C66B36339C147455A9284782332A1E5D1CF7E4842380545040284B17257941542DB2B63903069EA45A80EFA3CF2822CEB46CF2880681962056E2A768ABC9485C342D168A65C1A894A33C4D639C96C15A40FDA9CB4DB90AE202B8DC951C76C6038573345D05FAE89195D10BB23B101A773165682811DE4B5B3A496A298B40ECABB5DAA7D8A8A38A0C80880472F37A51419D0283FE6021AA26082291A601C2D77D320A765B07A717F609BB4AD0CB6A78A6EDA5916C7287D842C38577C1831B1CFEFF039DAB02C34FBB5D9A2C282B94E0AD95A17563BF3B78305E3C58960CBCAC7ACD6830674B6A4E74BC13CF7A0FFB8BDCD27CE2227B71DC619D880BB71E78CC528B609375E67229E94960918809B38F4CD9282210FBAC1C2FCFDE66D6B78FCBB332D4953659350B37A07E12C8870EAC6E238CC0B867E30", + "m": "3BB4FF5796B3AEA03E27E75AC0274C077C473CBA776C466F5F3CABC7104E5C25", + "c": "5F083786743B50C1C7329900776F47237978CD0CD1DB311CEB90FCEF3D89FBEF2AAF2E615A3511F41F97AB64EACB6226D95632B0B52ECE80FAF344A8968D48CEFEFBBC94CD43F5158CC2A715187A6F559FA0A8AFA70656ACD8B26B65B480E0458D671E5CFA749BF8A07C10231276AE9C221F124B69B04D5BF6A79A11DD4586798E38457A917132E3F1E1065CB316A87AA3446615D8F7420176DD5DFED430B5D7EE27781782CB6F5404F67C28DC883FC2081A9559F9539B514F78A0C5D020643B7B7E6FA370A2BB78C4457128827B0A37B417E4905D40F81DF77B4E9C661F7B5DDABCDF0BC5D2F5CCB1F1F04B347B97923E780C209E7695945CCD09B2FAF897615A78533DF2D77D360DFF3F6EB87B5C90739DA0143CD4BB8337CBA719F2125CE716AD018EB0564A662D5A08E7E43014137CA29E60F0DBBE5D52FBABE6F1393F6179B0D9A73B82E1C912FD1BA6DDAFB627269B5F7B7F5E08FB30A332F0C8E745EF67B4423A0DB1115CFE4453D98FFE039A60ED3EC093304B210A6A620C28CA0A1988BC06FA31A798649DE00684EA7AB95E19CFF366289AFC46421F69C1DD9C8231C1DAB1466E37E2F3CE0D4675BF1227FAB1271DF7B7F7514CA84CBE81D2C9C1F532D7D7C70D1957B67E4817CB9011B5D0868FE0F71F1DC8E16BDD99F79941B4FE28E6422D8E8F2F4EE1CBED346370AECFE139685E516078238B1F9D07A81CB3C99F8367A05D341A0B372FBE75B6441598D708E38F0C7F1BAC8230CF92390E569DB7E578F86A3F3D0051323537F90D97C7C6D2D7289CB2B7B7749AA095F214459561F19F7B86528A8A92ECA49036C39C01A6C745920362AB0807F7D7B32EB0CC3BE2C1B08B993AFC5F8DBD7FDA67CDB0F4C24A60FC636F129F32178A6F6C8FB2B5BA7D57F90A73174E429C50078EDE7580988484BEAE1639AA0C79357A1362F5325962E26F7217FED96226214D980F5D6704DA43785D5D49FD2C0F395741B61059597DD2E051C56BF9F6FF8AD20C89A38E7402D4B29EAABA32A45AACA0FB180C9B164CCAF3367BA826F515DA0AB639834AACFA8ECFB546E4DA3ABE03E5911DFF9376154D46676F7F15CA356B7CC6147BB9F70E391D96C69547207B973070B93C0171B295BA42656E5DCFA45E71E1AFAB9C018CCBC30F21156F4D7E681C44F422C19F1381FD3476E0CE282C022D8A72D47E45EF2EC9DE55BF4336BA338E8788EEBEAB35770BC16A9B89E35F40599A50BD75D9F5AA0A21C85DE5193DFAFBDBE3FF8765BFDEE4C0C1633C545099053676CFB37931F4B76803C26A1B4A39B2F80158B12556BF132BA797F8DD2DC9DD07A819AD63B5D9B80D0D095149E041AC610192F71B840AA10EE140E39CE45154BF79ADFB970C5E57ECC5C927B907D0CA8A21C3A03E456262728839D6CC756A0AB2D88E9427251A8EDF87FFAFF457D02FC8681BB841EC36AAC0EA42AFBA6F105B3828FCA311989DE28524BADB5F8DE9AB3B05F0285DC4656C99ADC73BE024F9AEDD7C3E7E410BB432D5E37F2B04943D6FB9D5E718AD9E26E4FC0851E7E254A565902D54A8F9E0540B2F72E52CB70F2FCA674A5FF3C87AD7DF3924E79B2EB777099CF37EF4433C1F937933C4E9AD8D99445FE266D0DF317E0E04AAFD82B1DE46AE1F83A2D0824FEB21FF4D1642073E8F43EE12DDBC6B7CDAF0E231A84F0FE55381782F8E5AB72499D7337EF3B0495865CAF1F75CA0A4AF0651CDF8216A3FBFFCABA8509B8F624059FC4DCA843EB5AA7BD99567F548FDE232FD0E68974CE66A1A998236EB8F0E7B9717A26C60ECE9B74BC99175B3EDFB813A193CBF936B0DAC1D9159BCAD65E4B9D7B045A6D126315F1C9902084CCBB5B035F92F5089AD174CF30F5BE5816637E1509E3AB79C7AA12580BF9EB6CDCE6C5867207044608BAE91BF59F3DEB5C338AA338C6C51EBD8199A25C4C7310556B3D1FD826141BED32FD25C289B5E41C78A156432A5B25DA5FC067E7A353C8D185F8180F656820709C010D7AE1DC93CB94B8E29F152EEE733F103EC9F74F35692E350F7FB5366935B83DC0726B90C56CF8EF1A9133066FA3FDB0319284975E4B4896B0649FB9495E2E8D854B37CD56B76A489E4EF052EE59141068264452316F794C0BE7C2DC596B29B83E70DF96E85A33395A6137767B4A4F09B1F063203FF91AD773D7844A60EDC008834E0A0013478A6499DAD7266E17125CB1F4328C815B4", + "k": "D37DE8FAA872CFACF7ABA45349B2EC3A4BBF0EE82BC223C8DEBEF6621F6B3A83" + } + ], + "decapsulation": [ + { + "tcId": 96, + "reason": "modified ciphertext", + "dk": "36BB5D4B256DCFCB14028B1EFD5A74E5606617950DA40B2408CB2C0462A222D4373A7A290FCB2997DAACAE414DD8C3AA7B38773C79C949059CF15B12FA8C522C538FDDE905E798A3E7D5A6F27C93330B8243A12983E5C83880971827B01C2A03B4FABE4B15071AD124A5A00C69D98AFD6BC63F305693B50040595995E99AD375BBD58989CDD2AA92E95532EA88336C5DD9F6C4A1B254FBCC4921B96DE593C3134A2219B7A192E87309BBC6EF95499D594B30A4661F809D03297B6BCB2EA894B724083115B0C6E6C2AF2D239AB69574EA95A746CB23A0DC429EF42A79234CAD8B9E85597BEEC196E85870CA1415CAC40213B64CA34A8247595C89FBC19AD73B715AB3990B7E9691C9E5C767D0780E4E3026DC172494BB4ED87311D20896571608B065224B2C08D3B754837C0B8AD8CDAC7775A8158A0FA93872C972F4048DE1E9C9A8E33D59397E8DE188251A6286BA2CE6F258D1F8A99E78C464F576FCE80A999C6475EC03BC597014DA6084F2C97EB88795F28CDB27437FB11F67977BE1F865722B85C3371D4534CDDD396BD6B12298F4287A7CAA87542AFADCC70A222FC1830182831BD1319D980BA03937B2CEE6C2417A766E841609D8417E9C9D8DFCBC956217A5EC02DEBA6BA6717A0C71009AA663DD4CAFD5FB5142EA14FB52C72FAB33B0070FB38C9C9670323DEC00D9EB9F02F64BF4E65D86C2C216799093040F6FFB80FDF6BDCC649F1F39BCD2C613221B528803A75E16BB5B8B067EBA7765738AF0647DA1026DEA6717CBE4116BB28DFF91C224243C1A72350165C30184AD901A9425538D255B72FD834C8E0090CB0289C2F79EF7997CC2AA5928D453198BC392F6524A88B5AE67B96A1C1782267F044A0AA0D8A011332CB8D34E1C6CC149A4A6A3B99621A308818322FD336EDFB867E4495EC9D92B4BC70F7B170A0D28B3EC365E5DA02C7E50373B3059248A439E0A7A0B3A656863785CA749E7419A9B1595EE28B7DF838F1F2BAF02A065B1242C17DA7974C5332BD7724C77644E88549FB572BA8964979979B5A7901CF05823B4B985E4B3D531C21620689F036646D528DBCCB4645B72EC920FD0F5426614BC469664A48A9EE0B5C7D9A52164E3163C299441E11DD246731180C9BD70219171712C7BB371B213FC976236043F9367569561CC00F033D473809B3C49FA3725F111907015C6214C63347C4B8672980DD856E30B55BDB7B3D22554242393AB54314715CEBB43AB224A911B090DF1CC5E6950504ED063BAB0A0D72A5C0FA8A1824962DE9CB685745659ECADBDC3B335627DDB5959FED1B66108AF4F5548F151ACBB3336A94C59A0A6B28D94B9359AC7C766CD7B9015C3B99EF45280470C37E985CCE280ACABE67AFC54151239125845B94FA26656C4CAD63B0494416464F4C7FE5936BC284E9AE388FA844EF71227D1AA0E872BC6FDE92F894C26E885087523CFB56C7440222150A62BB27C067173025BA2953999BB3746094D546A6AE03BAF8630FB3BC6F57239E7A35C20691FBE1611E6C46EFC25B77029ABAA947D81C286C4816E24F35D883B4D315A529A593421BCC5765B34EF6C4D416C3CD95443F9BA2FA248659EE77AA201987F6A5EBDA78D307559A874963B1927198B3524518ACD6618F2D25AFA4B11C4B396C9111A19398D55079EF62054A812C37FE81DB4565F67038B9EDA7A4B63B640252701844917E8CC7B3B1EDE433D0356A2C0EA7E2DF3C237498231A65E8E87874BD78487EBC1CE800C7D1B56476C220A8BCFBA568BA20B773C993A98302C084B2BA25297049841355A16FF098C420A2220837BA6C89B44356B810576760316254646B0A5A41754A18AF17D998541400CAC0DC830AD9616F4F17840A9CE2D69B3BBE8CF9D60B01B00AC7EC8B1E08B3D886A804B4A4F2FE30CEE898E42109F3A16C91FCA5D1C328B90F00DA0996F2365BE54927D93F83F8F319D8FFC375A5126356A94C6B6C3DEC98CC9F2CF0089B3A6B878C73BBEF04A4442B1B0FBE273A994BFB6E88EF627B6FAE415C37691C9D30E10B45A83E02AC3F16A5B1249E8B15C0304AB6F3482477C44A6FA8B5DB797FCBA271FC1544CA333A6144E72B02E929CAB8FB6011EB66399B8AF76CB188AAC3EE43B992482C4585934A5075428FB9966ABA67B414114E379A05C7F81C351F6E18EEB8C3042E4A8DAD0CED9A28CC492A2573590B15A914CD100BA119939610C6CD77D73FCB5BBBC50609152B3946E2D5969BDA6586A7B26BD4284F41269DEC3C124FC3C73A0A369818D9A491931607AFBC85B54410444F9A426C9491099CB4AD40EDCAB36F6924764403ADE37288FEAC93C2544F16ACB41F0355E2491FD948FB8774A62C0AB9FD3BDF48948C94206E386BDF8D463FAE547B580B1D9531D48E11D1DF6539A7911B5577F7D8B55B17547306CCDF4224950776A2C90ADCE2673983C3FF286260FC6B08138CDC93CA3297873AA9443C7165021D767D3C132B1A66B413B8BD3849E8F6547665601C776A558209FFCFC84EF1415DCD6091F12C790F1CFA2C5CC9C170BB7D0072AE997FF443D853B021B1999E1686AC6580EA6A0A946A2B417C17EEF967A51CBCAB67992A16A8C12623CA327AC7FAC5E4B616844E28F43BA45AD9252C695C4EE6707D60C10085917A3D5CE9D46CC9EA52A7759827FD64571B7B4630554946796B308AB65680FB23091515683D1CB815221B294FA403D79124A9989EB7402F9990712E91D78C89AE45B33647A8FC11C5770AB738A3573E2C32E485A07DD0A009DF27B4456834776ADE5C454E5C70D00CC455115169E8C5D9AC35844E5126E784596B191CACC1C8FF6C2C0E371A81726930808CAF3680CA194D0156BB1662BE1928D3B3A432F102EE9C8BA950C3CD05B394320C568474D78DC7E75862A155CC73CF2A093C87A96653FC1B1288FB130CAA41500541E34725E612C475740079BEA8EF227BF1269B9F0F86E2CA929C3D611CE52552859C7CFCA9BB0F125E2CA1BC1CC67EFF49E27EA6B087700B8723C9DC0175A424D1578458D0BCE007C5858361279B0C76E62A03E1977B880552396320976040D9155DBE7A4AE2AAE1A312B010CC430B310CDA08EBBE690E0A102D24BC508B2A31F61171EFC3C80E0038C20469436776C65656F189A972544529CCCE3D917EEEC4912F779E3435A890000042468F93A3D1F104780D26FFAAB53AA33BEA1A3103C3A51A86C6A7E178BB65C8145F6A41EFB6DCA856FE4847AC574476DD768271031E2E8A70220CAACA92EC136120DE09FFA31C034C0A243EB48762286B1B2218ED2B34C551572B6BF741A9D0D8A31BC1678067A3FD13995CB559325B29C39A78C7798C3806682D3B0C810F5740D438A1861C8B5F11343B9B21A11694BB1049DCC0C872B2575911DE0E8997BD19F2DD7B7D2AC501098CC54444ABCF1C266721122A6ACAC20B17DC5CA3F9B5089136E6EDB6061382E96D08B27D869003B8794BA83A50B1E2E72910699BF02513B35A1630F0BC2E8A666087787DD3B0B10529AF8499F6314039B827786B71CA5DC706DD0610DD220EAF024D577CE14F80649B86EF5F839ECF23F5C595223A294DD9BC0CB7BB59F039082E68B57FBB325911C2D8CAC1C694CDCE01D9E890561B43FEA7BB5E2E00C6D3A650D9838801810BDF5013A8B95E9D76B329C1F2A0A40C4964FC52823E55B2AF6412136314937F98150CA4A2679444773B66CC679519815746B74728A0C9B8346187106AA2185669023A1983A38580505D04E4847B2AA2A4127125013DB519F141B5FE640B06935C2B673819A665D9039848803128204C4B223533C9A3F12C9553C8A4161490FCC5B51CA2B6A81385169C38C78ADA8BA7C42877138A50FBFC5863E4967DE01593597CFC7110A0B9C82506C9FDC7BC4D4D6B245751CCEE2119BE688EA0985C5647294A3B9ABD07296473B70578FBDB0C4C5F16EBE034375818A63E408141B57129093FAD761C54A99133111707656811A2351319C04B99259946FE197B7E89283659623BA26185F887680943C1CB88341420C6C32CE2B5B6304FA99BD35A2D7CAB3E39060056BB4D756A78050835F301AC355BA7A412227027DFD2C681440AF9C81664A9C4DC5955F4BBC2479888A7536A9199199AC6710B495949CA82BB9B1A2B06823BB1A79C6162CFB6A8B8A8893152928A169B9801765DEB2AB890BA2D32C5C4409912C86C12C08972CB3A042BBA1B76675987279067C288FAB89DDD93A75368888A40C2C9A20B60A0FCEE8495DABB859B797C7B763CDB9CEB405213B7BB45F134CB6CB106CD7252EA069FEB250138018FC4A5071473CA81C6A495A46AEF27210D556FABC5D12281CA6D03BEC1E62E762B589499CC3DEB96E2A1119663DB91A6A308E2360A02413CCF01948541C54D5589829053E8513A916F58E7318073CC06A308A53FCBA7D197E3A2DED7516D56C84BA0B51B7FFA0E5D77342681EB856686B73D7942C37D8E2C8", + "c": "4F18ED0D9FF40079EC3A6D67F903ADEE4D5CAF5B4194D969CA222537EECD10ABFF3CE7CDA292B2F632F65A1C73E8CE44B6D7890E123987A8C08FF23F7345313294FB6B3E8E382EAFB352958637A6A01E0839DD1218F1BCA9167894383BE7F651AB0C0E74CA9B19DFA157DB8656E5C4367DED62F7B4432F43F5BA02F394271D79F1AC3DC4B1CAED50723730B337EC50303182ED297AA3908CC28EEA64E80819CB1509FE54302E3E03EBC39BDF0C00A755650AF3610AF7834FFFFEF5E6A2F7B66D68634E5B1B76F2482652B785DC8051494BCE4F41867BAA07EE88C9680E19139EB9BC5128FD5A348E01EE2A7178B3B29BD0C6D6096A7DC3059DA039E2AEB64D5DDE1154BFDFCFAEE6CB6CCDDA5039C64EC3ED3A2CD58FC4F7A4228995CB240B1BA31CBEA59BF6483E100C3816BADAE7414F0E744451FF1FA0912D5C0A2472B1B7C0ABDFB175424DB48AB6F7BE4DB871788AF54960D244035D0E3FFA2FDBCF143C22001D4D7D3CE3BA848562C9CEB024D5DFB8F07689E72B0F2AD31D05890BF96F46002E204EDA434511DAA17385B81E0C513CC36F9D53A6ADB36C1B23C2E37CAC31E5997E1AC3102D4FC2C3A5B3BA3C9772541FCFFF57AE162F5F55FE21DC47A66DA2AB98233DE8EE3058635FE5F3315E7BFBA87E8DD2280930240945E972CC29CCA19060439D317C146DFF758FB183D66542ED00F2841C203095E9020D0F10F416C0B7E641F4B02A9F71F599E42678C85CEACE3EF586EC1B4C4C62D21F2CC75CFA7F58A525CB6A55868EAA31043596E2985D52F32938B304678A89C72D3B91AEB6943A4B34541DA42E1060F6DC03AB7D6AA90B2EC7A4D8B2E1FCE969A8614765F8212A08BCEAFFF50A48C2A522E1A41FC2C2CC89827830D34AD556DBCAB5E69121C22DD035437FC6AD0DD95C0E22916D5284E833CB84286DDFC47DCBE9758B980D78E4DC294A3CE5A1175F7ECBE2D103801089CA744F2BBCEE7212F64B0CCFF10D4E9643027B942134FF2B50DB9084D66857C4DF62B2C6919D9DE84D51170EFC7942B3871FD536D26310FF39AE099E0FA562FB13B0418FFDFD398657259A822A458E3688B42D492117F16E291F913566CFD6CCB097C0C45C2E5416B7D3707E93FEC60037937B3930D1BB6A6CCA5418C1D26083B3286D00EB0AED6C2B7E77028FC70896789AB29F424D91D393F1F63ED8E0D02D3A19F16B80481CAD1809BF92DB925D11651E7DC1B0499E1853865B708D79446166A1C25EDE6FFCD277689B298DA836BF52D4A885C6E8F4320AD031FFBD676378093C0F854236733B002E55636D44A5DB9ECD7358CA24A3EF7638EF03B4B5D8E79DEF61221FDE892987860DFCED702D4CA1536348FCCD99FA58C1031D934CDE4593538474B2EBC8248BD4C4792B2B63B16A152970007574969F34CACB80DC978240C9755903E9F7951ACA608B682FA60A90A8282B3E54862B60CEC8EEB737FBC31B6511052856083CAD928D2E41F226C89A3DE0FC36797714471216EA0FF8A3DC1EEE64E24AFC46A16A818B4FB212410975034F5486A4E84078F59DAD65E5D1EBD37021EEDE2B459041D4250468459D535CA974683CD4584B62B45756EF0D2C0E0F2AF7B946597DCB86129279078C4FEA6563EF1E378BEF179D65B1C8B0432660AF5AFB461478B9A9CBA980925E8035CAFE83C3DBAD397F2BDFAAA08BA05C623115CDEA9A62129D271F237D33CF1266B38F122CD7A365956B9A08D611646CBDA81AD64CF096344F9B905386E7A231526D4C5DE966263B7C8299FC7AC8DAE2CC188AC205F60A14AA108C394260A4630FCCD65AA9F56FB26425B02C6319C305B042EB9E10DBA41221299BD6DD4E02CE5E4085DB6439EF118BB45B6D4C086AF359A3C4B79E9B8BCB93CAB682EB050C745380B65E8775ACDD27F97E934A059B3D663E35BEDC977FBCBBECB3D10C2D8AE2CD8B1940968FF7CE054EA7EEBD0E81E65A372CCD133E098867EDE82DE7C58D99E908B416584645710E2B013A849FC15C209931918CBFB0FCC35D87CA63315719CF8DE8025682F123B6A4288EC654438C6D75522A7E0B80F59BBB44AB8DFCB5372597590AB7644DD1ED74162F5FF389B417981FE46B27396508B28965BC564F8B45D68DDEED1CBB0B3A50EAD14087F3B9E5D8ABD007BC38258D127EC34413DFC4F41EE0B549E106F77C1696DBF04F31A2DCA4691E71D8BDECF4F2721D98E1C8", + "k": "5D1350C0937B6222662F3D0A6194B8F1B60D4AEAF94FB8426C91078B6C12041D" + }, + { + "tcId": 97, + "reason": "valid decapsulation", + "dk": "97250E3DBCCEC512ADC4CAC174BC8F95F32A2A180AD65C0A4005A86B79BF85B47CECAC4F0FC8CDB5D929F422C866AC88EBB91906F26B6A077125CB9D509B24B16AA7C1C39971693F63A83F91AA8303F107D8A54A793BC1E178932314C0EE71A7D1F1A3C53623D66621178067E6085F37B7C2401A50BC50ACEC5814D2F620CB80398918A266293A10032269FAC992205A276C141604639F62C99C557A46338E2570CE54232D7171BDD4CCCC6B0ACCB26742CA100EB9B2A52BC72E7587677C9B0B9C43C16BC56093141D627B76984C6FDE04984604243D286431C8A57C312BAB8CBE468A593613814F6739048CB3AACB9C32A9971938552FDC0BF056B1DF2B0901482534A255F44A79C17B00C485AAEAE57391564FE127C5693695987B67B11A6A4043876D5A77EB4163FFD0A2A538A703DA5E1C1696B1134EDDF43B6EC158895A612DB5A4D993CC09E43957C76F1E24B573478C1B4242FF389D0195B1FFE8A6579828462CB7849C6904326108027F16D8BD5BD872DE80A4D2E77F3CC3001AA69E4D471981E731E6033A3B66092F215146508BFE17372AB424B3974A5F9B0B4C76C9C600823A919502A1414203AD248BAAF2519E6C03647FC9C82A3ABC26796F46416FCAB55D4FF73B64393564C7682BC0703417CA96C3BFCA97950CE29A459C71D9D84A61F02F05127F2F9A549A7650802029752AA126E07F50C2AFAD0159B6CAB32F773B8F9CA6A49A0FAFDA807C62667E4B9EA5075CCCA22A9F791B83A9920B62692BAA3A9B4AC920F5B46EB99B6F869A06769B04C20C4A4C75D35A131EF0CC0EA9B973A7685295CC4A8983AC1A2E4ACC8BDDB86A11EA9596E8A9EE338A6A8570E88B28B35BB0D400BA83C41FAABA7C2636217870B41FC182A6D1A109D54983047A29D372F5EAC59129017203B1BE7566F9B518AB8BA22EA5B8F824BBDE8717F6F559E379981FD68A1FDCB57E2C75BA0C13FA2B598977C9F4343A945A3B130C306A8A8980BC75665504403413D4805A4AA77DD9A43549971BC8D3204786990F7C0470F73CD716BE9E351FD056B28887CD253445D81115BFDA50026779AE7775CB160A4B7379AA5A34BEF4488B15A25D177715748A46DB5C0CA94203C652B2D20737B1ABCD530C61D26F69137C344C5F030228C1646E2E567414063800B268425CB0E008B98C36700D186BAE0763CC030029CCC68CC8AF4C1009F5C4613A3B1066C12E90C9C48E19B31197758AC2A69F2AAB2BC942562B5D78C0831B3B7F44211973EC33978A3CAFEC3318B680BEC68BA5544331EB38427BAAC266B67B663840671E150C9992A5B522517E96A97DB38B241DA868ACD23676327211F460560196F4BC881C2212F1583005C5530C784EAD51A3E9C745579B056E4155CFDA14C6628A9E7B0202563E59BC084FD1BAA662B8A38AACA8E4CF99969D0C243A8895320D80A3948512D9AAC9E0D08D7FDCCADB91A1436276EBD8AC88CC893606A181CAC857435F262AB36B07BEC3A78882B81E533A79E5E54C8FA49D68F033AEE991EB8293E8022D4344097B06C0FD5224A019552510709458A620B98E8A6656D8F75241B1BB68E822E96A3C02339BFF99C4553493C38ABBED5620E2217CF388A97729223D00622ACA95B5CBC1E2C075F1DB5493767130D2AFE4588636851A8676409D1B55B6577650A7BA26DB6C885B04A9169568BC5019824DE600C1E947635F30B5128392A061C739590750A3BB9E975F8159673F199CA8824F8BD35BF1E062CEDA8AC9CCCF3E0A7413F42C2F65965230247EF46C80A08C50584C887715E4BC56EA478EDE317C5EEB0FC4573B62FB29DC795A9069C523B3CE6D16BDFE6B440EC6922818B973833F7484A959E7B3B981CEC0A136D4581429C75C6211C29A29CA36566B0BC6B2023B96DA991A3AB520988914077A8B67B70947BB4C2C3974C458A71272AFE03C459CEB04F889487A8365A6648C759486504156C6334782CC71EBF06534DC3A59738651426DFB527D46D469322777D752A96CE0586D701324F13A781490B475BAFFAA763CE00B6D9ABB08C61FA97A73DDE3800B365E61B5AC1EFBC26586623FBC1ED4502E8274C3C11626F0E01DFC5BCFD4E768CB7C69BDD493FDA01C0AD476804289B3B02185F75EA26A7925949C66F595D90B033FAA4401590FEC389C966B4F9E86A65394569173C309AB4BF7E92D2EEB21E8D12733DA1D5173255F0AC080ABB7C143A0013577DBA999B37022F1D7538613B03B40596E8CCBE92C52CB8961FC8A9431D88A94658BFE906B2ED295C4539BFD191931B89DDBD6063CD3372D6A7E17C74CFD55C9230CC3C7C46811436C92B66F05D88287878D74BC7F644554CA5B913E5391D2F37425F697B604AD9F0CC0E9AA66945386481A1EF177809A597CAC153638643314A9C1AEC06F175918BF8B04AC9005566B4F8BE608A825B3D393436F09D0CA33A5A787B88D7242F9C750C998A15D99CF27C6B0A5D00A8E47A39E09AE6DD8B24FB6B01E35934F493CDB273EC8A6ADF66533C2E777AD639E7167691BACA4C3B4947B465377C35BB10043ACB420D481ADD110A38F0A900A23B387FB78D8AA4E40A32A2E02B8DDC10DE6377194E51EB01B8C9A2A958E85BCD6DC7C21093B296895EE921F7853183EE1BDE3A9A9DFE0CC95DC9536897E69982816F5CEAB300B2AE5C85384B2516C92EC369A4B6B7DB51CB45261AFD2C86A4C3842389B80907532CE76AD20D9A2E850C07983C1753315A1121F10B1C66CC47F7F30701D12CB4E486D24C66754C3B01EC86EB6B056BA77636E4757CCFA11642212754426E6374E95685CB9728D6F357E3EBA3ECF241D118515D9BA4096E55DA801256EC13D182034DD2000DA02305FC282A4B40594EA429BD606261045F3B87D7CF06040C71D14141B4B4150FCD5AC3FEC11EEB9A0EDB3CFF1E80B1CD915B7464101958B8BC3788D3440C7F47F06140E5AA3BE8BA291FE2BBBB91671C9FA077619C6A79968D2E8768BCC052CDB1657E987D1C693BE262E75BA2C088BC8ADF9B7B2525B0B168FF26A5AC77C81ADAC2A2EB2BB53D97B2D78680E830A36152678661816372973E257D70655F1E9C2DFF95449B4C7511886D7D1430677A12CF1843B771B0FD76FE71081B7E165AF37CCE7D0CCFA71124D6A15E100A66E1A8D9B150C91BABEC681C2FBD96A33884032A8BD462305DE0327CBF63A090C2FE4B568C2B54B4AA5B3189C609FDC08EF2803C0DC59254CCB4A956136F19C63768FAED91B343213876ACC0E141AFC30A36865BED609CAC905632872680E59ADBBE76332E13F257097BB4CC8BB4BB278DB331FD8440F0A64C576BB347173FFAB65CC351AD2DABFA822542A477ED1455AEC01467A5453838BB71DF38C261A72887210C4EBBC6D762867C8479CAC682C4652E248A6677132BE56B1F9C2B74E008A878C44BEE92DE8435197583513B409DB87B29DFA32920CA144B437738C70F0C68A58202979D66FE8C50BC53C2817605207CB4FC4784FEEB3275D8AA4A74687FA776C55374E67EAAC488595B5A1445DBC28541CAB5D324DB2D967C6D371AC144A9CEBA82588040BB0C8D4329B81A97EAA42BB649A960A424BABD31DF9379726807EE1245532E0BE252563ACC336CB743F698839EE665F4E294123767031E30224C979786805F369B0C7227732ECB55E90CC6A42B08F652672BB414472AB094B32887803E6DB57DA64B0B9094DFE29154BD0B853E0CD052C2E534902E448561293CA1CF60A24D6122A7B154AF59763A48661F40D11D2C3BB60AB306021F5659E3FD196ED8B59C517B892C457A8F8BBB806AB31B8B7C3F8377B79A214002CE01CAE9975875D2A9CC660AE8F386103B7A276DC883D145EBFE9363D0290E5B63FB6A04CD28B83D8116102C73DD40B21B2E00958924ED0FA33F0546288E6A453914CC7A182876B435026442D050FA19C3BDD451598003F5892A156E718802AA2A1F4882A9146FA78B77BAB9CBB522C3597295DA57940396CEA70B62982B41C07AE5EF33F591A6D1478C19925025D2437DD60B99433097AA463D136C49AECA8162C37F9290C5CDA4F0EDA410C580A203C25BB0C360A99771CD852B73C3FA6FC52B2909D95529D9EB69647A42892390B1CC3BCC6F9C86AA109F8732BE976A85C98401E46C00F5AAC50286DD648B11A3B4523BB126086963C748592A7261ED9BB65E62887C22290F2C9D3E86F289141023A47A1B09B202A0448B444C6321424C5CD8AFBACD04BC17886541AC90C4FD600513034775077B4037B09AA11FF687EE26AC5715B3BF3430FE98C028617CD0040441985AAD41863F819C95A54940F00498F0439F5A04B518506EE96571D9508E5CAA574C6687AE834F8781CAEB17FED8C8721AB3FEC93EBD097A0DC10A493436080A89ED15190D6825C6F3F4DB7B8283E1A2DDFDBFD4C8A3779BC55965A9AC9AE909A994ECEFA6279ADFC260F3686D78AC8C96E95F5C61A739D20DB8C", + "c": "7A0A63634CD704EDAF7FDCB2AB069BE8CAD96BE23DFA96C02BD4EED0BA60E43EC8B79A668B6EC2259D60B748CE5C2BC428A045FF2E6687127BE9D14811213F148D39ED6512509E5160543D9B737FD5C074D5273EEFF6E4E5B09508106C18E604FDE7C927D25DD4CD2F76B7370F58BB6E0798698DB855BD687019FF5C9D2AB6AC398239335D3D4BA465AF994C9146410CAB5376D913D13078C0B101151394B7F5847C90CCA38708C3D4FA257E6144E0100EED9899601802EF337D7B5B1A260273EA5B35A5F61FB48D13FCBE43869E84F4F7CF1D1B6AAE61DBE95D7FF6DA587CAA3D0B4EF865B49BB3149A43A9F89CB76387BF363B0C7FDACAF4CDAA851815E490287CCA86F1F424660294630470A90468D5B39839E1C275A07AF30318FFC50EDC34A24FA531196735E78BEAAD162057E6B5AF92170D1BEC86A415243213FCD7050A2CD34E1B2842C762E13C39666A950964178603627DFCC2DA0A583CEC0D2A51EAFF9E1F5CE4FD58CD239DFD91D3CE683AD5E8D3005FDE4E349FB2C6DE5292561531E868AC91218E2C123747BFE23298481FEC2F127AC884576C6944C79B2ACA24F1DE67DFD178100632FB3AAE44D920DC68E913E82D7D817CD44AF11946856EEA887F321209EA08F394CDB72108211F70B1836A7FD6B6422E987B7F8CDC34E3B2F56243875E441FE050A5E2E963BA804E02ADD36F524F30B6D0FBCA4C8E833A52BE516AD1BF9D7089D80CDF48A7AB09F4175A1906059E7FA6DDFB704243D2E79AFAD7F46C02CCBE5A64DC7F8BE6EE79EDD35DEACB476ADA83113A078A65C96889643EA2BF0F2B297DF76D4EC9E9BBE9FD93DD839FA0827FB2133BA17928FFE64DCE9D74AFE9CFF2B7979AEE75B6FFA85173E97D10A3EC16462BD0AD16550AAA45D716584FA747992D4F3A8E2FB0869A3435A26F21B773C88F4376A8F68757C7678FFFF04C22D566DE7B3146532B9321C1A6408601833ABF2E5046FC4A192E6A0BBA38399E3B7368AA0B41B9886F397162AD1DC321B92C735140A2808250E08DF557B515D3235F35B540FBA3B76747D0377B517D24FF1C5148F62D54F71E54EEBDA2C0AF7B2AC9B420DBCE59647B1F9CC00D16E5D08F6FDECC1F4222C75C19C4023812EBEC86E11DEDD337F15E18131BB5D856AC19C5F7E1FBE411336818812B472D55ADA4A9D202322EB6270E4A2BA7B1BCE1C1352A84518EB456B6F570E428BBAC4B5653AC529327CBF312EE23DF1E72D94143F035A9E4D3C6EC380B0795A418187E5C90FE3872E0D1F82AAC2F206ACE6EA74590BFC42CB05957623D7C02202354149EF5E6323A0D968D715CABAD07B6FE22084F74C8DCA6AA3E0A40DF4C8D85811589D44718B53760359FE2C39F95C98EBEDC70F9832C3A70815F99238A3A9A29C479F31FEC54F91D742914A79C9F9A29889DC2CF159264C5D26BB9BC2B9680A1D2835090855D680F9D48E5CFB82ED8200FFB1FD8626CD7C6BCDEADA75025F55F184F32C2B40EAA11626C873501DC8A3FD1CFF14BE88726E718F1736E93D9C58A67B62E5C952A3926FEEB0FE8080F198E2F00D77AB5788E795CE445994BECDD5D4EF7A282EAF323C81A39276AEB9D4F7F7B11D22904A6B512145F176C2BCA6371178C18EC9B12D233D1B10553FC5262358FF7AED33B0155A79D7D0D637D1BC538D6D5817C83EDD614170864941B3527A1F615D18085258CFE837BDF73520D94CD7ABA32A48A0A80C36680D7C3C8D83B80950C1DD9AEA2406B755F6CDA567B513563514F9E71C1582E9B95FA6E05E3059C2EE2E4317E5D67181EB17A2E65812C6824D2EBFE63E664C06BBD40D162B9F28371D61D8AE363232EADAFE8B571ABE21063FD1E9CD7915D9573D42331710E17EAB4C268BA95532C5A8A7C7CB2EC035847134F8D8260E9A760BD755C32AAA7BCCFBF5A20DA39ACD382566D2BC30176E80B8ACBD129DE56083784DFE7CFA8E36AEE1B911CBCA6101F0786A6791BB6025B0D9C92BD85A426CAC2D38C838062B3208C49AC06F9DBF4D436965FCBCFD3222B093DA213FE641A623ED3DBA61A8FAEC417C14D40CFCC2636A12901633CEB3DC0290EC05C5E7AD3AD828D7F379B1347E11AB25E9F4F2FE8E4CCDD15B3B3ABF961955138849C348D9E9C6A4A5CBC55FE22DC87790BAE23E350FA8F4D27A91454EA4A10589CC95D09066FE7A5E037510C2A6CC1DD0592037619C6CB6294D56C5A", + "k": "8D77E1681DFBEE67FF380AB5C5F66A0C5A6D30E34CCA71523B4ED7DFD51A1A56" + }, + { + "tcId": 98, + "reason": "modified ciphertext", + "dk": "9C004D41EB2EAC59AD6CA481475914C5F07378B05EDF3B7FCA5CA844185449C61678953E72CC0BC5559B062B0F63E29FDB7209A2546DE4B21539E5948A38A42EB0CBFE8325C219288AB7ADAAB772AB46C7FB071B93F2356A22967867901EE7605C689FE8CB13C693B08A3AA302C2B707E4A93243C2B933083A04603CA05A9F996DC205A2CF4A20762BABF970BD32B8481F183D5A8C7170B234AEE2B50BBB88B52AB3696388AE0AAFDA484A5178B015CBA4C2AC29539823C384B45C65AAC95B2496C02132640E21FC04396C696AFA9F77619DE82448B1530A42B5C455072E7462C8F7F35758740C023C8040341824D965ADACB445D52C3FC91607770AB2D8508B96C407B7A104021F6461447A730A38551526F09BFD462D4733918FB588D5CBCEBFABB458111565F809977229696B8FE173B3204556031ACEBB514F49A563711B0A3C637594CBB42EE46FFDE34EE92430F4C41A2EC16F68180D44D21A67525BD81947F8350E5159C9D54C0F295642D0812FC11611CFE3797D7B2B34291B82372E79B6A5B120737297376AA30F6C91CAA7BA4A8FB4173E46AEB7A24EB315443DFACB67D067A5D93AF9C844A32A9048F94F7B2968E35956072CA5C57707A41235C68B8A971C790427CB53B12E32DC5556103F0FC7919CE0A79EF6991A92349D5663E5973C2DCABE876B084958B0A63C38E3778537E86B85237E5367C176E3BC2F2029F7B524B499B84C28B1EDE9B9345BA5B070AC61586D63D79436805EA2EA11B353839736AB5DC384B997025F80C4EF65559F0370AD1B5D385754E806321B906C307A3684BBBC919B01899B5CBEA35D5DB99ECB38A7943C135E49564A3B3C29B18E42F018B50B9C6B7782566209ED93166C121FBA88585A165ECC7AA69219C1DE9C4E019677621B3870997EFA96909A331AB2AA63FC704B86D745F8228E5EF7C6D8E242AA285A06321BB1FC55F36B47BFA22CD8F849689AC3691024537808522816D7F71914B6115E0B71F9F41CE04565926782BFD7C75524AF6B8B63DEBCA86DD97EFBB7C0D73946A0F24316046F23FC4939776D9D9156CEA01071283E24FC02838448C54AA5C03A0AD77B563805022E9131CE77201D746567132B76707FD310CA512517081C2591E03133FAA06E97BB58E625591BA5B5F901F068728008A8188A16E3034CF08C21A379ACAA610B63E5AFEEFA9ACC21191EE618A6777B979980B893B0534C44FBCC770E837B34D60030C47A2A790A09D42C72B4BE53198EB38BA88839368095C375552FBED00841E08F2DB7B991393342D22F3AD6A9F0FC763EF0B5706ACC402B6526CB9DB1EA65AF79A43D184B1588481475BD0175404DE0A266144867924F748A3D0C776062856627D2A21F39CE766CBBF64358FCC86ADA67769583273AEAC70F7277AF501299628EAE607EA4F38473643B3B983943D28CE3411495A88413B531D9D200B2928AB9C6324A5621F2C3352C0549D0A5C146C763530611A292263547CD7B2BA8C54510B9121056A3470D2B5E09CC7CA1377D3FCC18CE74B2C6E5C66B1B3D73AB1D7910679FF821A8525882CBBAE2FA65AA20BEAA78C555461E2D331BE364100A946E5EF90C8ED74E917C79C0E81B3F556AAE5C46FFE15F4C85B6E436BFAE3441AAD9C0215C6BB0D2970E3417565BA104249404E5817871309BB14587A807C402ADC1F45A6B927CB04CA885951ABC6353974214D4804EE7B281E3857BE6BCBD97E47202F5BD28A71E3D179E0475C262B489D63C37C5630CE0A210B4E5CF7DF19B81B1206DA4AB9D367A793194E23122A37823E09B35F718BC9026A26C970F9FE6C0969A1E75F6ACA3255291FACD3397935E349A50778E690AA734E7778B6C4EF0E4915A0106A2D8080C676E00B28BA090101561B204C1791E00A064595F6C7A5799324B4F700AC41B1B1A923234638C8597C5D3D2ABBCC6CA52595248036F9D277706785ED2951FB789A97CA061059245EA249C297CA7E88456CD8A0A0FBB27BDD6A71C0049A6C86F44792C7B470E6A73482AB38A94F247C12A10E546AB184AB0A8E97AE23C1C758A86CFE4BDA5DB2D1D1C8C5A1739A0A4503D82246886CF3CDB3369D2A51616C43D2850459395726BB975A46FBD8C73FDF726008013B17C98D47332ECF0725AB7722974C62021A5FD479E82F442D4DC6C768B171C83025F5B0D676C6A9882B6BED90A6E861247EA0A96FA4183275AB5D484695A1C3994696F38A920225CB2041D29D096DD3645EFA8350337542AE09FC6C5CB49FC3E74BAC9DAF5BD853406EF8CADB206B810E31564B6ADA0C3547E00A90A04793A58A824B4A489E1CF932C1FB12268364B296ACB615CF81229CA33569CAF28929D8FE52527967267606E96E49CE4A11B13E12258D4846B0A2345F053A53C4D4FF14E8970BC3C1B614F074667A624B858C54147B674D7B90330A09D67A31E5961C3B5C162082481B9354695778CF09A1F0A4C33182F969039820033FD071E8E33C87BD2BEE1050A88B86DEC251A976C3D4D3C6E5613A7987B44D37A8CFEB5BF8CE5CB6434284633922FBC8F9A169EB8F0BF20E637ECD28F50C940F2424186722ED6BC9E57547D3DA241F5E517683795B8757275476D511614EA405D29A1CD70EC0FC7EA90DD80C99D4567FCC58B91AB2544417188A6CB55C8CCFB212B57EC1B023668C5A043518B7E17082EAF7A1F60125E938CB7D600A746F10E4A486DFC96A13E50B3B0D166038666C6C02D9DF624754AA6C4824809A6BAE60736F368488AD8949523CA70427B57362049A3958DFBCF27A0A6A1046355C1A4FB2503AD7098A472A22B3AB5E53A91B5C4912431A3D9A18E0EC5075FB720CFC0048830127F78A96FF9837B8716B5D88675D7696793B62F5524827B7A4DA1034F65A8D3D509D95A51CC51BBF49181CB696977B2710C0B9EC9D945E3647D930B207F7CCBD36B6BDA2586F312073A020CC27185CA26411E4750DB354737143CF5540C3190BC225B49AAE549E296C39EB6CC6C28079456174BE7ACFB8A3116D25AE851A116898ED38552439C28F13B83FA78BEA8BB001F8785D0859F9C21697FDB3A7BB54859384E7315C5FD0B5BAD34917F2C3BF1827578354B97984A8A53360EA58CD96753D01502BFF21F1ABB31C9A7B69094761EC957AE08965C4936E9176114684294A631C0E12879592770262B9BD56604451EF6B5548145B316F290A1D2A68D1743B760CB4C373321D977DE48032D20CBF61BC6788C43FB1701E4F65E2821B884E44FCAD016A8865FE0CABF9E682773734B7AB50CCB3697D74441FEC87417965F5B6520591B6B1D550776D1CEFC37A1B5D12CC9E462A23332C79CCBA8C69989F60897696E738656964131C68417C1E89B0F8071C6511D1EA47B6DB37D314A63FC847CD625BDB6E8458C5A0B8B070AE7D1A2E659637D9073AC1B6E99A2C1AE520FE6C6A527B6A64F329672374505D773C0EC11343C24C3E71E422072FBE7121644009BCCC89E92239D7C761D092755800713E821929523BF6A1F52C307857A22BA9B54146B15EA159403DB8CE3C2AD57E665B5190C23E9C3F7C4B90A566292F9C3F1B98AFE683418615727A876DB6078A2B40524E325FD881625C76AD5BC6874A47BA562BEB7D11B46115CB3E10A8CF61D3AC73FBEB8945F4A3128107238268701DBB574C217E618AABA40AA56678219B66376E429975B4BA6F18CEEA451DFF657A6182EF1DB42EB918071F7C0BF1C4F16450DF1823ABA8649B772C273D310AF25BE7F250B6ADB5898D74D2C244BCC05744D074C579825156738EC2B8020DACE305117F173B6FEF86243193F8AC81D782B36F4B6565F22A3525248264C4BD8162E724B6140B1846F9BA0DD8464B2ACBCE11447E6581891C98AA39B5B5C4A299F473E57FB6AA5994E1CF71EDEA78F28C77C1C9B8C00A15F4CE7BCC56BCD3E6075329012A16377A2B2CD29D7CF97C7B34311486589A043C58291DA6AFAD42262A192F70C24E09ABF4D9272907B76BA832711176711271D7E9CAA30BCC5B0228B90A12A5CE182F1F2046830A009E0B6794844A301B6AC1646AF68C2F6BC16B094286D212C3353A26C7A7C6935A8CCC43156888FEDDA0971768F7E558C099AB5AAC79900660500883E8FB078E91120648AAAB0083EA0CB026DF22F82547E99BA0ABFB1741D075DCE1A84A567C4ACB99CF475BF4DFCA20E0650EE278683B6A2C5115024C7AF7EC16C7C65074B8535A13143CD590DD4960B8048498EC1BE5A48BBAC9A5D48002DE4000BDBD8898729C40362A9B04070AFDA881ADBBFE695A01D73B8D722B6C73B0D58A9401B803B1E882C5BFA1CDF4395D7B28EC69614B1C80FC74949AABF5523B52ED2F7DA22260EFD872DCDBDD51C1980ED2CE1524038644E72971011454AE50A899899E840AA421746D93ACC94FB8BDA6619DDAD22F88D9336A144491206B47B3EE659DFABF1E99A66D6DF3543DF0414FD2DF7BC5CC8EA245D0FDB", + "c": "63B513E1172CEC1034C6A626C8018657A5EE6BDDF4C781186EEB42D676CB998E00589EA7DA35756DBA56CA139FC8FD1E19ED21D41DA8A486F4FAB877D38A8FE4EF97BC51F338B086557004D7A53FA4F281DE3FB3FB2383177DC3AF358C6377FD4146E72B5E590C9C1EF2D57B5B75BD84E44D6A8A3CC9E1107C3F642A227E1C35C7954E2499393A8F48CD616890D36BD6A274473C14CA6CE68D84594389FA0D9EC8936F6B1A42AF4601BC909D52B13BC726F5BE1084C7E63D5FA65E16B7D1A36A1C38400EBD8B73C76AF1EF838395ACB2AAAC1761D114432F7FA5243B5EC88BF8126F58A9750F7CEDEB69D732FA12EAC65568D64443CFE4CF60B392C2D1D23A0576165A1AF54B8F21CD145D897084DC863B52E0AFD618CE8666D5F551498179200858607BF1A7FE0B4C0BBBF62E65CE86CFAE3B2E7F2FC8582491E6C8AF96CEEB56CAC2320D2E16DA251F6FDBBE40F86B4DCDB9FBBF80288FAC98686A7E0270AB991877D0094E6762D76EEE5817BEF9718F0976FCD7FA9F56F07DB47A0F0B1B01687F1E7364F99613C9845E697281F29843F2F40F1C1CF0A8BBD810A8EBC35A9E3B7598ECA61751DA922078DEF662A8B80405EF019CAD33DC124EEB339A48BE48FBE703100281E9B4AF4060DF20F45B7E48086059D7A8FF44863EC4BD12F7E6FF22258F7C00E7DA2364F42061E06C4A3C3E19B5A33FDC0E32C663CF8AB2448C3898B0E052B3CC51635EEA77183C7257A6FA2A19E463BD4E29946B79D41EF8E8CE154D7E0CECDAF985A815DC194D1DC436629F0542DFD5A276928AB1A225959BE8BA5E18873C188D1B07A55296F831077505D4774D26DDA37086A67239F134D129291DB08CFEC921B4F4649CDA8F45BDBE49E4456B5C6360B932B628F3F749D0A4059FF98F5ACFB5723770614D916249065534F4FFAAD2F31DDE7D3FF50E538EDED40D68FB0F748FC341CA5B13904D2DCB9FAE332861FD81A1BFE7DBEB4187CD10167D3EF990BF533ABD2D9AEDD8E6E0B5143FC89FFA640FFA1DE761011C8FA8E30B87299053A9AE2BBCC7540B82A78DE99CBAE258CAEE29B0AF4F75FF8D7BDBC7BB67D789B374B71E467EAB0A9BA4ADD8D3F90F6A2C12AF642F3A5FE2E50404D9F0A897A4E04CF1D1434EEB4E3AACA845000A27092C3C97EB2813684C466130C673FCA1A3EBE17D66D8491D986EEBD4152C9B2C21FB70B4DE6A2EF576BAF76C5256E630350D7EB43F164B274108D9DD72B9D3624FB9609FCDBFBE85E253EBF7725156665F9D05D4C0BFE2415FA31B282B71DB83E6B5CDA39F3632B44695AA2F894F7403F367DBF26B117031BAA296655148CBA5588EE35E903047E2960C5F70F65D4C5110CCA521A8AEF81A743F8130DA7786C584BAC7B50C14E6DE49765DF8D2F5B1B4611F95A4B3031E1E7AAE759F3458560DFA9738346BD11400A9E460A74DBB524C27D70C67390193E4B5E515972FFCDC487AC26D347250B9523BF8E4FC4B3F88774B4C1673A597C7E214679BE9E3C2B640C273D163F8654AE1304CBF1A1FAD2EABAA1FEBCE15F0FF808530DEE24EB5257256328BBEA9AD65DB1513EEC31300C3A5EA37A0726DEB74468F158BCF7474759235E74DB6874F1B1DF9EF4D29E091CFEA32B9B81D29B51CD6124358CA2BC47A0113F5DC66BD0A2318702A2133C7A43B8106CD2D8F08971FF205A0F2FF4193038A6AC8902D9A7C980102CD87A6603AC9C3EF3005E886483EA3CF7B649977064D4CC0D33FB8400825FD715D69A20BC76F24E137A63BCC05BA801760F85B7AFC3901D7A11E13F1E5E14A82C0AA5B01DC79EB44844687748C057347EE8B90E3D5B2ACB7B5D81C9F084C8068EC744E3172A043E4C9DA639552BA91CFDF52603D3876892250868E74465CCF735673D71B49E95456A670C0786AD65FD45AED9B51A334DDB2DEDB54A02B2F6C43EA82B2C93375A5F1E165A9ADEB803C91BFB06B85230663EF16DE7FB608F0D85CD8CEC9E7C24A863F18C4E6ABDFD2382FDBFD91812A4B8AFED6B688E76A36B08F38D2F56B4E40D5B8E95AFA86C6EC8CE017863892D07385B2240DC7F10953F68C1440DE5531E089EFA01390382BDD8962B8CCBA235D4BA9A5B46B62AC1D7A3CFB87B6904087DB8EE01E1C455138F5E7E30773813012F5238AD2A35FCD160D685DDFC1CDBE5E6B92F398C5265188CC7916D48810D95290A28A7246791853551659B559D32D", + "k": "AE84FEE1F4075E4CEEF6961D122E621C1380387FF0804B8B40230BD03A44AEE4" + }, + { + "tcId": 99, + "reason": "valid decapsulation", + "dk": "BE5BB6AC09CCFB2235EAF05CB409CBE8325C77803BAE90031C515F011C1F84002BFD5C5B02728A038566AEAC9EA6E46264B7C2E4B0A3FF43359574740F3A1DAA95C75CF38D4A2ABD9716909574A525702C47A060F6B71176014F9D46C9E27AB84F61805F4922D4743900F199079A23AB44B296746B033101F049AAB632A1F66C5E7308597AA18F901C25DB736A8BF929F5A2A62F7854C9A3A3223C87AA497CEEA791B2A30D24F89596AB553BF3B38FAA7D65A129AE411D4C526652D9C7BBB0A1DAF10BD3F392AECB7AE6D7343BB03B0D949737A66C8D4A2273876113DC4F819A3BF03745D8B194BC1AA81E5100D8D968780B3965D733A6E700EC389E895A9B17F7A4B71159F4D119F439829C386E80D7ADB2C7186C476E8D518E634C6F4CA3706F150DDBF30F805611DE98A7CAC261A922BFE9709004EB7FFA58C4EAD47E9394805A1C20A068CA3A379E15C351A6C74D40308A5856421ED726BA2B6A82137CE5916F868A1990087BDA16658EAA00404253AADCA629586E5488218EA7B23A87AEA4AB5E929C66EAB897A917C4124521FD346CF0747FBC70294702541676186B99665B2425F94AABC9B1B350647D24810B63FA4E8A426C34578FCCF82B4C78AA7735C5A9EA35D184C75201A17389B92CF1B81E5BAA77710DA55B7B6ED3928035592696004B17BB8B0759A3718847E30324B46ECB655278E7B9B7EB216D93A180C90A4DB307CE42B070017870E05F594693061818E592C782854E289649BAF48CBE72534D0213811095FE093BFA5A0F2B4A5D1D839E287552A0B4034D03BBA97B487389B9989A883B80B133537EB24659C5078870F575A45B36EC77877F6A29A2C2258C876E5FE664EEEA2D169366AF13B8FE5A33CF9CC5E7B69F0F92264EB11FFED321D737B15D5785F1FB129F8A7F25496BBC6345F66201F5C30196896B5CC324C17C3E9ED290A443C833F73931212CB4A7B6EFA27FBBD30036152EA110A84DD2969B4625D38443E95647A2F96046D80A1A1A2D2530CD0276A96868BC1C89442768BAE7F7C90B8003224B054861508092A029600BEFD93DE55403B5539C9BC78DC6624743F5337E3394127B9DFBEABAB8079F228B55BE1111D1F1739EF3CEB3D18BAC37980307C539C81CA5F8387766C33544C1059016729C9EBDD1CAE1E07D0D38BCCC982F4434C9AC9244071CC17EBB4C99A47F7A3A9E50658DB071524215CAC3D7901619C78E1CAB61D3AE8FCAB03C879A2657A04EF79ECE305923676015592392501453C3B91FC4A7EF3830D9350AA3C3ADF1AB9C25A113F0BC5AFA289EE9DA014F644ADCB76B74235CA06667FC0B96B90C3B3C9417F1F5CF9B57B11FD76A3B82A6DAFA5D3D726159756C081623B798BCD7F8CABDAA927D4307D30884712A4BEED01012166E953645B6E966C125432878606B742586512B1A22A67CA632EF391070AC9BA55582EDE72A2A96BD60EC6A290A6C90D829FE67C75478C4F07107C06335BD460E4F1790012A94A5478CC65C4639C32D191BB9CC86796A7836A79A5C0312B4B48148351B74F2155307F2635D534153599332021CA09ACF38704C85427339078460430A8E7754E8749538759892CA40CBC54491139ABB507DD6E3587DA493DBE949DF1C26486BCC89C297768891032A8879885336B29267997615B3A81C516C8571CFA556128D7B953713102994A585E05FB5764711A47E14D19B5CA22C26B42269279B5B9C677B172DD7417A0A26B8DA380704143783E53FCDF3C0CAA63A225B1C02A1838C1C87CC1790FB938FF8837044FA971C905E1D99552300A3DB613A302951CD1A4DEF352978408CC71C3A22DB01AD796DC813A65AD667FA3B9F8E2C5FEBEBCA8DF29D1FC00963CB204C1A2B5D99CE84E46C21C54A722A69E486B126433C2CA46702ABA4C30308C1B816A0CA2931D8C901FA2BE4CC8D6829B12E6C6264F73D3FC47276AC3241E4A00297B44FA60A63F89FD9F3893EE40285D33F4FB92EE9B03532D9B851E07ACE25061895C9635A76882C6FEB98C6825BBD3B9A977EE7515DEC149AF335E946B6A094280CACC2BF4788941B8605EC87C74C8076236EE98B4965844265697674E72118B909C189762BEC1B5417345177BAA15515EBB0AD15698C9A103928BC3E54B6413AB41AD7F9AE2C485C6B227A71915B6551BBCFB3579163607F60C9F9949FF36568812691C3D78C0C479E8AC18E13B3C59F857FFA66C5AFB1B24B40537C138314D3B1222418F8CC172EA801660518D754982F14386AC80D44742F50027275217D4C0447F62C993EF4B95CAB8865F46387738C0C4223929A70C268BAEE9243C9112C3F3B1857F301E5A80015383AB613004F07110DF16D40E36AB1EA607DB891A6437F1BAC06A81544D576B635D20DE4774832DC88803AB77AE3CB54E118BAC11EBBFA1FC5BB2797557B4F1351FBE9AD41331818F79901AB93BA74BD5142499EA79980042B39C56AF4660D65804BB8438FB3186E536186F2613CC6820F3D60012969BF41CB75F0A422FE945899C7CE81D802BF4742C103B96638184C9014C399B2E13A73256364E48B69BA83651D3077D113481BCC8FFC323F2E8027A738A087A98CED88B59891AEA8FB4980AA6F88E38ACDB61A0671289E2C374DBC929035182BF801D8473E7F6589878A413E4B8A0E39CB874717E2F453CB235495D36FBE534D4E42368FDA4C78B024311990698802BF65581E2BA7C2F9408144069EE82C63F866F6E6C21A45B1C438B5AF9405A7B4CA99C76C6E632F4C41359B574029667FF8985AA3F090BDCA6EF1C35E0F4A970D62551AAA316793825CA936305580B24A9CD16C9072C10CFEAAC1F435246E26B8F14A37AF662014FAA5EE7075A4C04259E18235124D00DB7309D10889BB33F1D69F124C96D8691E0AD01F1D371545C953CFC3CCF7965E1A5B19AE0CB4D3556A5F9B15CAFB5F6093BC9DC7CBDB1859C91964C3CCBEFCBB409663065E524BB208C6B9C7BDA0480862C72EBA89A6C7C8B6E8168A7B8A09969673830C634797961B758C9FC4378C129FA1D311B9B6A36AC87A66E8568949C27CD0851C28942C0BA6A53686AC58359FA76D6E685B3BA03EEBF988877A7C594698FF7861EFC176E265339B792B316B1B79804B7642534E20C8E0394B86089F75D278BBB06A2238C769C20C9F65A4DE827EC14B62FB262C11F63BDA4A8360153157F34ECAB00F66D6B57423762B81549B00AF6FABC53D0A15106AB0ABA083C59B891DBA2D9DD737A24B942525041383395B9B8379D0A1E63707868C2ED3676D508C5B85AB89B9AC48F7789441AC3852662D1F175B51AC1821582C31B83A29C43DD63C40DE62B223A3BD034BBC06945A5AF207F4E3CB6835AB8ABBB01C547A5851A1E847298A092F8442A183A559B9FC62C285ABE871BD9AA11B5FA49FFD158D50076357F2900D78B59623169861052D0C915B4C6FD89097F65A2EA61BA8937A1ECC342C9FF98E6F87954DF3832B31CB6B4ABAE392997A580EE6781A82345B4BCA31F960629B3BAF9B1CC1F49B27FDF27B7B504EB8C475F5EBBD5F9B0F4B3AA982CA73F47505FD0B41A4A343BCD42F48576EA92A72C1F1558334C5EACB1B3D905400B4C47D091A1CE19488911B8559790732420178AA98818B144AC2A1C47E6289738123BC0F8A430741C7136BC2DAE3C556E8CF4AB6BCDDE9AFF4C1B452103F73A1A94EC0B6E4ABB3F0C6C774206DDE35B3A1812C26505D098458B5CC1A0EFBC3DEF77F881C9BD234C0FD81BA1E69BBBC6311EE029931A7666FB17B0BC1A3E98642A289C0854054E51A79F4C94D2DA8B808A7B68A894DC3473BFC05382944CFF7392649C1A8F929023D9015D037A202A88CDA2821392C1BD13ABA1F7647C78909A412BA4306BB2D54C604A6577695810ACB717224A25DECA5448368C112BB084427134A73268267B62C46AD4208CC97C6B314810396CDEAB03C81D538E0F7314619BF61EB7435D21C01DB2B5CC55A06206CDB2114CD71B443A048B9A04BF2FB4FC7901837994E11309005505E8B7115978671DA352792DB359A727B48E027ECABC015575C1021785A31374A362E98A756409168E8B98164BB3611DB385FB5A92C199F51078B0F5C8F711C105816183C7962D8B58CE2CC29B3F7C3446296FC69514F02620D2C567C992621A52113EBC0E44C54D4A113EA5497F91AB31AD3701D603B6250B46D695709CC1D21E2A94FCB5AF6656096C84036085A57AB5331DB7E18F504B4807639816CCD535BE93B9746893C0D833F325741602B5A73308533B8C9E2C596C147CCC59829244701398B03CDFC6DCA65176F3B06DC107BF7B7CBD29B28687424BB22CCBF380C7A7B2D3D7D13B90443E2794443EAF82CCBD7E63CFF57398FFFB5A4E7821D0EEA23811AB1EA0E64FABA847E846AA789F4D09AD7F0DA3AB64C560C7120B8511CF5982191F103CC45042180C7495480D7EE2F431F5ECE88F9A4AB826DCBC0A4A6A4", + "c": "F0BF18E55196F7BEDB00C5F10C013E83040EC31BA323A6F5A62A989E1319993F0325E19718BE8663F644134D292DD6946AEC85433B94093E7CE9FF26B9AB56ED55630846CEFF33B5EEF22892C5425CFA566D9A182EE3FD7463A6B78F56684598DEC8B4F55CABA267A85C6500A0745F55F5EA38D73D484B8A837F6B9F780B4964E495EA41D480F6CB77503C16936B130447C8944537097E30C59BBA795C8CEF67D7070E17196C6176DB38D67F5C70198DC426733906E5E4AD8B3594B12F9FF14241AF8B18A871E851A87DEFCCA5851D84BF7B2B55D980A781F708EBB8B814CD1252A97AB28CBB86A7CC475D74241D4E0E97FC5A8CA3DB84CCE3BF7A1F519F1E1C5304D788B698B84EFBF6B77878B0DAD7ED1958D76E2AEE8E5333BBDD6C824D4975CDC2DEAE906ADD868BD6D7AECD00474B7EF6385F91658DE4E0B89B43F52B79C527DD2045CED383348B85A99B03AF6744A4C0992C5D0BA4D20CEF60468C13F482F45EEB3F7B2333B133FADFB2B3A6CC87E71D05B9778F0FB2FC485CB2567A759015D7F3F373718C8E41169D3C2FEDAC7185401D4A1D638E5479605177148A336760E30059F049B21AA37E9E11F11BED5775B24FDC294FF2B0D114FD57114191E7F537F9F7A2AAAE34B008C167D0825349B223834C23F2A8738AD549504409E5FD2AC80CFA6DDA79A39C5890D46A2B6F701F52ACEE18B5D975CFBBA9A98AC4CE9745ADDDF309DDEF79C450E51D152AB3A7F4EFD172E7D1A19CBEC737917240FF5A18A82863B45CC3DE3EB0EF97A7B77B8DBE7B5768ED8FB91DF1D608348FA06C5939E0DFB7D96DB3F498E67A9952C717E9278692C5291F5E15FA0878ABF863F10EBFFD7DAE5C014EE8F3C4F2CECA74CABED4E3DC58A891F90D57384B9E12722464721B668183AF94B8EFE12EC099B294126C5FBF49118DD2D2FFFA2B374F49BBCE64D60789E22B0B4A496180E0EB798481B4B58393C826258B7632058FB25EA78D9E72DFF823DFF36D738EC87AAF252C7B4AEE5B5CE2F1299A23F2745CE900E25EA6EF5DA30099E9590B61D0F61EE010071A7C443EA01C5C47F0A8C3A2B5BAF01A1BC6ED51F65D6C3ABBA5F500FEE7C613BFA215A35EA25FB9BA6BC9EC3777218DFB5BEC59391CE9BD30442F4B6DBBC4568E2049374E9B94E6FFE27323120DFFE0367F9CAFC67226A958C84AB58049E54A1DC64B015BB36AA184FE1C733B36E1E8682A8AB88956E28B323C4EB7940EA57771827737A5FF62AABCA8F43EAFB05433A838B116AFF9F7B4E16DA1E1A1BEFB4E4B2BE16F5CAD567D6B279B400AC729EBBA513217059B211E582B80864A93773264F6E5CE3B370A39A52C516E75004013690D9EB7B038C29811CCB56923391143B7774144C9CF031CF0337695EACC8406FC52CE3C6240769B2A03EADDBAB53FE2EE753EF02331817F8B9B40ED4AAB07D1B5B331A3A41F78871298DE2575E2F08627F18030B023CC5E15DC5402470B54327D8668EBF7D16F6B445BBC76E5346FE4F635D74563CEC32888F9BBDC3966DFA5E173CC5238CB7A8DC5B7B0C8D5483D719FCAABF35AF31FE3FE9EC7AAF8CDA6B749FAF8512DF81F6C9C14509ADFD1FFE3F4382A8124ADE2E0ACC6CBE96DC0147125084A243A77E490CC115E860AEE32E77C2FEE184A1367C2E821413A6F3FD7B3B50D9D96AFA917FC0FB5F49E0E92C07B8519A1C7817C6BFE5955FC183B087CECFE843937F62C4799FAAEADAE6BAACB7A54D0573CA025BF65004A8B9024519627718DE7D8C9F95587DDEFF430943BD23CDC9580688180C2EA5DE2036E593DFA3EF5010DED150F76FF11AAF9522741E8EB28416ED2495FECD9403D5106AEB42F35B53F7C714EBE4580086DA364E854C589C639DF4DB54D054E016BCCDA848863036E49E106C38ECD8FD89CF07A5960710E80061A0F429EE4DF749D5B54E16D378CB411CF10E1A63E0D807A509DBD898362CC3CDAA257B2559B94C28A471C1B8839C729249CE86AB203FBC2FCFC6168A188F6732C50D2F6357F0CE9463327D7AA4A5392F0279641F84EB3FD8D6B80305ACDD358C4CA5B338EBF072DCC16B6B1724362EAEE0FD3AFB39E5B9A593A5BB119F8E0E0DBB0530B01365DF55AFB81F39A55AD89CE31E5CDFC21097F31AB8B5F8D97F8532887EE9F46B558490A61349BEDFF22ADE08049F86D28EF43AE113E291E4248CD7B91385E43C07C1E99DC215E68", + "k": "3F9581E18A3B18B15E7A2112528B1C28DD00D14B3FF55889E1F6DB6C9CF79274" + }, + { + "tcId": 100, + "reason": "valid decapsulation", + "dk": "B3A0B7A4F289D2D071052C10AFB73E118C34E747312473A80F3610817020A1192EE4B48543AA1B3D9C131ADCA432C6575206C60F6A7E4CEA058B225F57E40B85D25038919B5365935A59253F1A92260A3E4A1448D699342E1CA3ED8756ADE5558375C25151B859BB141D3C1BAC33B788490A7DEB6D8996AFFA619629D1B62A01AADBAB0600F422238A103CD4812420AED239874E2944FE2111DD99C332573B02A661CD6023581B3B028AB05618B039AA62864784D1A093EEC71996E26C312B99546B297F330106F3A4EA054C9874393C675539B659BA4B33387128995C6275310708473EE72001DF18B09F28092AC23A978214FAFBA10DE208A79768836094F6E936CA457B28C64E9F7562FC9C6F9A8B08DC80C04D0909A939C159F920227078F013BCB84A30B73B4921066922C406C2A223DD17CA894479F7ECC2C4A3B21A4045AB712AD0B27D5D472798F35E03598CE9150CB2644ADCD78B98A902BE037D0B3287EF66136776690EA56130A31338176986E34C9571232FE063B53A3EB16C63AB974BB14A8BFA7806E9A0223609A0E36053E8821B743287C6787ACC1B8A03AB433947930CB6CBA2BC0173D1BC9490215E83C7501533B196775E973FB15AB667502F14F55DB7C2A84F8A538C97AB69AC69C19967D16C54D7315279037B3A722C26731AE8C84BDF137FEF328C1EE9433359560B2849DB8B050855BD09908E9039A9DD6ABF0970B9EED9BAF7B6B88509B35741AD7AA405DA1B8F90B57B349250810C73E11C7270911870BBAFD995CB77B55F3F150F49341D48F99A1448BB81DC048013B22445CC88A75C0A27045B18C1DCF44ACC839DA8B0058FFC532B94CF906C97D57B1AAA938E70293131F852EBCB99E5E64CCA12038BD335CA6077B70AA111BCC1BFF616C88C6FDB951A83118B4D3582B57CC00F724711F6139324A73556C150FC091AB532123910F1BA3353833EB5B653763AC1F4F2696866ABF3B117A0636AB76A6735FAACFACC3C98E924E78B52BF3632E2922CAEA22178A51ECB55795BA14854EB2272C259C3DB855A97C33CC5063A2ACD64EA8E57D938950A3CFD962989E39F0BC40C4BF7C36258AD94EC71A4D9A901361EC1711F9E1009F6849F0FF11547290E90046E814639F6E8503FB550EE4B33CE3A905CD7B94E752BE53B298A146585E6C96DA6517FCC221865693CB68B56D2ACCC1497300C8CBAE54E594A674BF489F307A6EE878A6809B1A8EB23A7EB8A9521CD3E583BAC89CB3B627FDB3923300C2963E1A16E352456E55A1C1A66EBA5BCF04473CE25687431C497679D899C230DBC43484B5E554244DF7CAADA8A991FE55A6C113907B8126BC2CAB1FC29B88182C1517789A8CAD1AB5D74A6C3182485DC42130AC14DEE77608A927CA38B9562B61955C51A52F5A0EE27440217C52BF63DE0C3BE81585E9636BA5EA0C507C46F0BF92FBBC3B00A448C724156A5A661C01364BCC2C145A535F5E3C6B738CD936087B525CF5FC101398959D7ECC0CB43CBB9D3993C691C01F12601655C96C787A8A072B53A63567B709AF66275200A32057E95E164B8433CD921AE4038B81E4554C80562DDC25375BC253B9534EF9C466DE52FF7461B55A50ACE9864165BBBF0AB8F323898A306C0809421461304B2087C41C3BDE281A92E69AFD507C8CFB1B3F8C4915EE39505B2C6D100922AEC77DB260FFF91AF741B7B013AC1C7437BC0516FBC1574FB75A108F02B2D489A2BC1C2D8D067AE3C972A5A7FCB50C6C6D1B7E047B216672A19B50691380ADE62409F609979E89372DC5061591548764AC2D58733A060744B31001A00E5EB9BE88031D846CF2456940BD525CAE38C73C16AAF5C4BB949AFB7DB0ACF67CCD3DC839CA9323FFAC08CC58CC92881E3623D8BC2AB0B38CCA3812701DA3950C8C93AEC5FFE9C8A30C7190226B045975193A71F52F8A24EF14E1EB430AE14219AD45F24C5518CAA547CFAAD30C7211FB84CD93057D65413B61B06C596210A662DF08768B6C28C91E2BE1D0CC978ECCBB9B92BF7E2CA4C6779FFCB332404882EC5A32A14CC9086CF5197B617EBB3F628006993985DFB5E79B3CD22A8ACFC147AC083797C994E14D8182347BA4384BF3F448E0026786C3A62A75A28EE3208AFB68C8DF9958FF80DD87B1550F047E7F63DD12C265D9A711B684D849A07911C09C9A8A76DA56B7F68611092BA9448038A76426624B706011E02C997DA250ACF89BAE6875052B8B8F5160936E80DACA41F7F491B7524709B1BCEFFA34091B4CE5158B129B5756BCA8664983311C42BB5A6632594B8A22C43913650CC833541304C7FA36657A9866A5A2C7DC1ACF2372A0D6230EE121C2AECC1FFB7B504A0277CC4CA5B7940608183F896362D0C1FBF361C3F6902DE391521D3BEABD549368B60E3A706508B06DBF215006A04E474AF0A862B6157AB7100B633119118790711A3339C9A77E0A70876761EA0FCB9269AA8388347AD13CBEA4028BD27B0EFAA1029BA3A694786D1648AC13323821A3332D5C5E6BA39C8E33245A50D17EAB9AD3618BD72816AA112114206817A4537BC7BDE984361B85695B6299FFA752985048693BCD6EC83048C3BC2C28224A6CD05CA30CBB13B85006AB1B75CF8D48E5E6C790C94742843A33F74B9EC329202270BB740B5464450AF379E8D9A28F30C29EA769BA77B5221A6C72E373FF9E261B93657BB692D420545397993B5E5CD0152651FE5CCCD9A3CD6C812ADB887CCD03D66D21241A74595D8996B280470FB258611C17EA36A2EB7404C8099992799630843E865A29FDB6FF4F46C2CB63052274F057210C6C3484EA666DFD2A4A2B877F5E21834AA1641F482A4A82742F72C8E4B19E5198F3E7A83A854BD3DC332338008E4336CAE30684A790C509736D89C0B3199C809E62B92257BB3433CE3423CD86B4223EA015FB16F35839F7378301636CD28B20DCE2CB871537690C2974EB5A2B99603D9E31F03A784A7C16CC62095BAB04BD1F38FDF30AFA74164F03AA4EDCC14554550C0D7C10D257A4EA66CF0B5BAEA25135E62A6AF1179478723E278C0D3D3A6BC6B662530012AE701C7971F0D418CEF6526AF4402C3579728825F00F748A8954776D36F7E235A758C5F7BDBBA638B4C44168E0566A0D73387C0183A7B2685DF330E9B346EE96966C77338E009BFC54C5550A3583BE2A4950C065E180E01321C4749A0FA08CDFE23229279394DA513F84758B82ABA850C4C5F60ACD449B520F46561F74809AB6C5D844A4B743C0A341C57DCB49E48616DD533F1460403AB58E516CC1E04ACF2FB26EAA80246016D46B844E7D28616957BDFE43D5B300912B3636770506273CBFEA7627256417C94A67E979208B318A3076F5985C57A6A495CEB7CDE0B1E36080EA4C0939C61C287AAB2317989A4E992D7866D9A937F587166F8A7B93091B897B1626B55C6954BB488CC276AACCC38F9B68E9A206703126EC53C4F493B671B14966489BB04216514BA2C2545AA53206B7B4EA1C9ABDFB2735AFABC5DF04CB7896345FB732F1737A3895BC52AA674304D9AD1ADE4D42E84B03DC5E5AF9DF9C4B8FA2948102B43A32A424C69ED42AA771B73B1FC23A2E5552ECB0B560B18E3C339013BC9E0616C2F3C6669C0A98E3A4312D6857D3802D7146E421BB30446A10BA90F111432488A3E21CA8D28D90B216906F3776D3CA82E6651564456B2DEF9427062594EE3AB4768474B8B47B31448C1E261804C45321008EF08C5B80CACB0C26D41434CBFB6298B7473E319154FF7B5A4413B4FA3080617CA180637F170BFE4D5A8AD4BAAD0F5452EE7A2BF932065364D3F274164B64E574C8423D8AD25135CC0B02EE6E63969307B17DB21FF72AADA431EBE56449CC79555C7150F911F40EA32436B798BB4BE2EF81192EC78E40210C587908DAC153D393C41BA9F541758E6EB2FEA8311F7F53D543A14F96B62CE52254BDC6432F4403A706A64175841D53A6CD08EF517348A5AA3AA1725B9868085F9A99CE9B4D053C50448CA69CB8529AA3384BB9CA089428A447EFC2BBC859C89BB63CDF0C4211148AE08AB3D1DBAC496A823C96B5EC951788955993999643349A3A4E7CDDF499090C3AF876760D6ACB2B5062CAF0B8D68114727FCAA92DB0E41F70F64DB0911FC006D23958A132FE5DBA81AA3637A240E59572C2A77008BBC99A994C6F7B35DFF652E19C64D52648058A2592C57A46E6B3F21771317028B0EF84546121896F734E7EBCB50B352A234BD8622A9E031B977516DFE991A13812F329074355833672AA8516483266793672967737848F684C058794344975C215BB96E14626AF52C882826FFEC4B7B8163B331A894973CECEC54D55CEEB88DD7CF9ADDFA9FDB40D53006A4062DB22311D60E74791ADF3308ADDD65B44D0364754128635B401D8846D91466D6E646B8FBB4E79ED1CB303303ED339B84920B4B6BAE9A506486CE67560757E62E7A10F0DD7238C0B4C7B078", + "c": "572CA32FB92D3840AB3397E12833F96C79B7B3B464DEE37308F07484289CBA75CA07D02D1D7BB3E7164AB6462F22DBD1664AA995BEA910754AAEFF85C0461CEA6A55D4A53CB3ACC6E9C8A72201B8E49C5C321A83D1F535D01D8A1217B9AB0ED6E64927678A266847A88D3F1FDDCC8342AB94BC6F3813634B0789B57B5B6E6D63CD645A85D5995E5D20374FA98483FD70030EBE04976992FD9FC31B9E62FCEE6F6F5D783DC1BF22795F31E96D7E274343615BDB6CF91951C423DBAD58B4667EEFA690FEC8ECF68D31B2B3D8F7E479D4E4ECB0416630EDDA28A31AD00921083D23F8443E35EFD8DF9A581EA03D8837793907336032079663B1DE0E32FCB10304DE467776E8A9B9850C18B2F01A058FBC554E3FC75FCCBE5B2EA30453B2C50FD8EA59453304472F46D955ED003ECE1C908B164D94983CC7A09D3AD09C907235E08624778D169B6728EF4AD2EE8C9E0995EE78C824F9AE4D51CBDBD4B6FD0E3D8ADEFAABD404BBC32F24E2465CFE6608A317D4A1E74B7B479186E17481D52F136CA5808514894499D4CC731C065683BD0D316CBCA851B2B8497EE6FD2C0E2E4D53EEBB4C51E893D71455E9634D4B7D3BE47D53721E83872A05EAC1C0A65E6E95A40E4E4F663C17624E06FD3155C4592CFA4A9B798F2E14793EE54CB9492FDC2515D927BD8D15ECA2D3AA4AD465750B41E4FD27AC120674E24AA0B7812D63745DDAF770387126B81F638F721C0E30C48C6AD3B5F6F43BBDB057202F5C7529BF978EE21F219FC393BB568F8E26C66EA341A9D1B1E6D49B346C5A4861E650AE95C1C954113F9F9F9DE3808D2828D166029EBD47BF6B51F0E7560779F6229EDAB36C3AC780DBD9CFA6212729923F4564868EEDAF361366570C2CB7C8DE2FCE1F860D762E113B00F13C2E848CD2FEAA90749DD4B87F088B03B79B5D2A5E313FFC344D5B5645224F5406608F7B25F96BE988220A128059039BE5E15E14F2033ED10C55AB52A4DF17794C323E9BE97B70BF71DEBC144038B9F98088453EC13350EB179A0CC1733B4E1E6E107CEA6C2C8CFB377D424462B45620B8392CC4B79C2EE8F79A1C24ABBD45520560FF052CAC5BB7607074FA1F6AA76E898DCEC45B210F6757A6ED0652351CADB0A884299DB15674911761277AE9F786A31E8ED198C4A6B212C44AA1DD636E69316038657C23F9E3FFDB7A98DBE6D6F0442F458C36EC49FD9702F12399487A5C7E0EDD4905252BB0937BB02768F648A5F872E6A97DFD4D0E726FC9D2FDBADD843E21660694E51A6251A776CD90B954039A2558A6AFDC31B744B84DFB74C309DFA635491E8A02A4187D4DA900944A60CA88E95053E30144A1FB5BF425DF49B17E74D936BE4CA6B667B2129B7BD389EDEF9F1CB157559F47D02DAB5FCA35A7199C9FEC142E44BE62460FA481BA298728F953DE87BFF32EA554120066DFA20794381104C54C1B73C636F6D29147761F77FC13EF79C08D7AE6858D1188A909BB4C28799391BB9CC4C60670664EED0C9CAA880DBE5420F4813AAB603E95EF25D5F0A765AB803B4947534F681AE4151F3743D738AEA29B6253FFA4F3963749118E198CA638036B4BA8955417A78683181811B29739321A7CCB3E4396B2608EEE157D1680EE57108572D8486029A7B3AD02BEC4ABDC4E3CECE793C20A839DA6E3854F428D24B0567647D7A7843E04D033F1DF7A7517FF30992F6101DF30F7B9861ADDDCD0831D1431844E9B43A3D05366F707889726E2913FA238008E7D6139A880E2F1FE07931CABC16D6EA85171475DBFDF15D200FB9DE2303D9AF2328032629982446494321FAF1415F2694A738A389ECE185F6B0B02E23CC8A77C860C40F1260A2B57C3514EA52BB1B0ECF135F84E26FB91DCA4EE6AE4EEEA59907EFCFA3CA018AEF97D9A95F8BA43D6CAFD096012A6E20EB322060D3426FA1758BA360EA0744F7EB0CF555094DEBFAEB6F0E114A76E1444C225611E17CEE1A60E6AAA3F3A81A690549F504D23F7092F545BD0E8757A4BA652A6353A9D78AE3C01DBB3EE75697E0B79D69625470202AE44D6B3FDD4CAAAEFCA6F05141F06A1EC14675352AE45D3301ECB765FBB80793BD68392016C33B16AE8BFF09885F0C540241C18550C1A9A8EA6AEDFAF4D27567B046AFA2DF697C3AEC2AD36165546334055BB1F8039435187F3FA96CFB5D29BAF07F70C0C09D3821E5177FA42F4E3F0673FD1918D", + "k": "6A8348F32429863588683CF653D2FFC757D19CE3E68213ED6B9B19D243530839" + }, + { + "tcId": 101, + "reason": "modified ciphertext", + "dk": "637578BC4CCAC66A6C58A9C0742A2B205B12BCE7AC72C4CB70AB619F1347310B1FA5D3B42E595C8303404AC4738CE3370B457832C14600162C80146E96459158681C296B750151C0AA883D75F38B53966E40231FE6223465A93E19C06621B65C58F1532F4B3A17BB03AA4294B61885BB1B1238FC7C44AAA96E3AA5FC5A84FFCB51B5995C846ABEDC84056D2A4148063A6DF2305B49CDC929B3A1DC06E301A0838CA44D70CCA092BE18192EE8BC7D0F661C640ABCF26B7F6F35268014786C0660B694A4A75149C345B08C046BB3845BF833C6945BC7249A8B776C65987505C8539CEB01453C04BFD299208E5A8DA70A1F6EB65D1C7C98CA700FE9C53156335E0F2B431FA76AAE962194E3A16F57843D382B2510B7006078CDF7AF67D709F4085D6EEBCF668869B69C168C6A8136D25DC7E1144EBB8FCA72181FBA5965769B54B9A8C2D096C2B48C560BB4989540D71C3602A82FDE4757EA2055E231452032BFDC173D204710B2F4845187BC859501A77793B0822B4AB619EB6840ADBB853B909AE8406F5502CD0DA0702A9353E90273E97A4F14237098627B11452956A470EE036AEA6294323CCD48E0279331ABF396C79268ACE9B3797CE99EB3E7C12497C6753399FE1AA6233360E9F7310E331909E83827719195D610673AB8E470B716169200D7CA199CABA7494D30494AC4B1914C8B127507C26483B52F80822D8854A54ACD9BF46E827BA7103831443428AD900C3B586D5D479342BCA48B379322E9B858671783DA28C8B4626DF9844D5C386E4141594539309099FC0B8E378C22E4F422FBD49375D994FFE70978B80305D94C37743FB0A25F89F637498B0987F21720B27CA863685F9525267308DDF1B17DB81F6BB73D19FB3744966B9CCB3DAE4336EBB34F6C332651262B4B504FEC3291FE4B8C2FC1A6A0EC047BE4152690AB773ABCE18C4134E88F11569C52567CC51738300AA7E8831752C728742A2D95BCBE620963C950BA6C1551B3047E8E606C2368CEFF7462312C5ED9F00E1DB04626A621BEB815DBD36778693542D321D4361DDD155061F0BDDB929401BCC16F086DD146C2E763A19F293D8D624191D59CE0148CA5B21F06347473669B1CF44E92930F4AD8C70AA16ADCE9C857F8771B2BA6EBC1AC024B741E663CD48866E099A456C980E0ABB660DACD8D66024CD247F665A082E90AE2F240014C1DEBA983955486F6A08E219C4D5F842FB00A7E2C0732F3646E3C08856C7C7C4DA2137F00B0AD20191136AB3232B78BBC008929416DB67AE4264313B39ACDB7837312795A464298082941331FF165CB46F76DB638A71B51BB254666715C5A5D83580EF4256D9657DD95266C316F2599521702CA9AF5514A65A742F91FE2A84C655318C632AB5A0653349A2DFE70982C726CFD5B39EEB87BF23C2F800BADB5D2A7C58556AAC0438D6A0EFA4C37991687DDAC918CA6C4A6D174B67CB46A72997A671B89899AF461206E4638046BA416B23635B624040504771AA7EC266B36FA8202B1CCAF11C3F2D25975A5AD3A94783E74C0CFF00796DA051077B35B00A8324BBBF85580E1A96D3E8C2946EA222AFCC75A8C4E17CAB31766006F2225722758403712A0311D53F8C1F51031BA4579D9824A0404ACB45B3C396A7FC95AAA9C6B5970929438FC3E1B89012CC17DE6497C537A83A5BB062B060F833A8D53FC9086EC1FA07C4D32C597A4D1621A142CA9C994DAACAC799866A92A03B51AB3FFEBCB5A522B3308CC44F1714E209E50243485043BF2464CE2340A75F477725784DD35C6A7CB0C7011904B505A0F62573D93AD17940A2D30445ED6A14E95566CE1076CDC4E55832A12349097518A9D5B11AC70147B14143D5092C14559528904688A9DCB0ABCA0D5A655A54820BB7D5D99AE2D966D23448EE75B678D14921B936EE2650670D0AC876C736FA00B1E6269CB4359DCD746C7F20292D1C406691A5978CB8371582F3A598557C7E1144BFC1C0C4E4A21FC83519BFC84B14649D9F89C9DE3C3D26B09FA7C1A7DB509C47320A3E593D37417739BC946E18A0C8591CF6BBF85AB38D7A57D7EB3305AB26F611A7F20092167651830F767448155ED845EF80C996FF6B1B9B95F3E75C7D532B4D6585465456666464E51502714E69503845DD270369A089594D30F6A8CC063707DA3408DFB8511D4677FDF5C875093B4F50BBC2B7AA3F0E3A39D750A41DB3861BA2F68A8735460CACF3130CF1C2040835062551B91124F23B24AF21B165939133815516C65AB24D9245EEA9875267E42B0C990A637D8103DBD6B6C9FF99B17B298CA2A737A44158FD800CFCB417479B7F0094CF2858E8CD21B484C62B78A04DCD8762EC1CAA5105A06ECCEC152A6F68B941487AC4B066665D3217CF629BAE42A3BA1B204030E56AB46A72C67EDE65F16404AB6B724AD049A99A406498A5C0553B286FA4F43103B494685F774A1202654DD953374D152AB51BB8E4171EBB145400CCB065456507B549F18CD63E2BCCD9AA8075C803F8684DCF796E186A65A624369812A3CD195E305A5724336813B9B00E615CC4B6F6575A6178986C62698EA4A92236B70A44222AE3134ECD74D8B693E9F615987C7554AC21522C59B0A939837B299F0B61F569B40D5C79485C46237CC38B7A242317AC1143B8C0955C0C348CD59C110D7247E35073294A693829B1DF3F041A845B9A2E3BAAFB2B1B6B36A7E0A9FD86A82567841E92A270B4606B90984088571B0B39D865B7C2105163E98A23CBB26BFE9902871B6D7691DC1056BEE661ECF874F8D63B2FC18926DE5A161775F807B411A0AC0494530189B6666C7442DA6A0C9479429618BEBF9C2409B81B223A326C3988E4A7179E79542E71DB9B54BD1209806128004F909FBDACAA7013A6C20CD0010409BF9C82AF008C725BE79BBAAB9A7128553A58C30B0A9C231849C92D83C2F7F65632FA4C2788A33F3959C212BB25B6C5B9DD733246C337259094D2C02A46C7437C7AF3EB9BE0F076215CB9687B6BDBF607DB3F0995AEB54B2822352E3A1E5CC304FC6904533142CC6BE5F8B290A1543E80694068547E094C48295C8BEE3AA02E1C17924248E4C018D8A364B0C9061B659B7F14F20B7AC291679958C5CE25A053BEA7815EBC6533ACD1C257A63ECBDE6A911C78B77C6F1307247C700A9C5A9AB830E604C56F8116A43647DA86E9BDB4B0BC6A139062491A51D0A6A405933415D51B445AB783E98CDA8D543B236748015859766403F53B7ED491492062DAB3050E302A3D05081930A3A39DCC6773A6E669A7622EB00B9A3A09BA9A55F2B127000360BF008542414F902C181008A3FC20724C5C7568315611305C10A7A3966472BD8A5D312AC9D57126A8C4B28796AB7719AF040B4C493CD41F6514E81CC3BE182FCE7171C4C9ED62912984ABDC2DCAFE1A172DE4240C446517F94B60A18166B488002803BC6610832D471AEDA1C28324D2C4846350A697F9267D9930518A810407BC32D5C2D45E6BAA2103AEF34B70F1A74EC3CB3F79ABEE7C0A2EFE54E7CB6BC9518485EA47AAFE9BC53F4A7C26A14F2E37FAF8201D680C8C8BA399D26AEC5492E1384C6DAC0BDCBF57EF2E03A9F7620F467868F2711BD87A00FF6C22D4784CCD62934D160FA8ABB06A2707ED0AB6C38A94ACA2DF24C9869D31E0C6C83662CA6A45C67E8FA837D37C97FEC96D0785D12858D8B6AC5C600C1F321C65C200501F21EAE78ABB46608258B12A073598193B787B8AE1B21B1B839B9CE236303A314330BC1600B44F20A06C4299367163378437579691D7D8250D11B9863773E1C960A76E59F5355A8F69796A18206345C4754D53A35C936E1A78CFAE19C3C30448A42BA7567A8239B9579073629B26E99BCC4FB69A4B4D39045D5CEFDA22428096317BC9C0B307820F189F124756FE79E6CC121FBC75DFA94690D5BABAEA200E522913976AF57841233C43C0ADB0F7F343F46F4945BC4032E452DD83499F031243B94B7E5D9ACC5A9B15A3924EC34015E805DCBA85A0DAAB340EC39A715765A14A8832541F171236446CA317A840384BA12E4C63BD33DEFB4CD7EBA8812021A4CD7C6C0E2AE215599E0B938EBF6A51B7C25E739BF33B68F05B85CC1D03617E6C46DC6ADCF09D0E1FC934A152B7FE0742E185D30E5C3BD87AFD5C3C80E113CE017705EA021F13AB9C36322DCB3A01CABBCDA2A6FD81047AAB12C99AA5D4514376C4356B018179D474F6D210F56AC8CFC0573B921C95C0A29240CB24AA07D009098DBA78884050C85857CBD043760FA2334B6A263BC946F96CA17A562E1A02D156558EB65A6599820475A2B18A12AB6E074A4D36C027A0AFF4047629797E6498A2A0A4486BAF743CCD8B1097F73498E6A6B66E3D27A7307FD0FF7686686FC9B513AF511FA3C0FA0CABBAD9C65872DA861A7D3643A21E4A6CF133142466EE68E2D0FD662DFE396971A27281C7BF595FB410928EB6DB1C162D2AF7BF0B5E06E30BB", + "c": "2876FFB597D4BFC600EE1210EF5E0F03FFB231BC973371A36F03628A6B1F4608A3F32BA585A36D4076F2420449A0E3CD5A69F1977491E181B76DCD72AB1541A191C5F81FA16FCE0DD857A217D9C694AE6D63EAF95F589A8CC14F4425912BC7CCEFB06EED3D663A94482379CD34E05DC4D48004939EB558766F06D9B6553A417C96F8BD8CE6BF598546AC1A978854D7A1EE1FB05E03EFC38A8F06B9355E4FF032329D8703815512BA148DF01B6CED36DF0C77029B4B544ECD5328F068A28D39B51EFCB38F392A63BF894FC278FABBE5C1F53543023B9367EC56FEA83583C576412D8BF1E53E4054326CA0F0CA65CCE591792A33EC208AE79BE8CE23BC3B53A32EEFB5550A45307F9698571D08FEA8B15FE8BE8D6B5FA467C1889BB5416ED82B62BDA11391D9E3CBF013482B652015DFCAD6FCEB4AD514B7A098FD8315CA4B4A0B0E93B719D7BEDBA3936A50F54244D3799F93BEAEFCB3ABC5A397F5BE705EC702EED709E20FB8652BD6A3BE6B46ADED9E8696E560D3DC2588026E9B311B9AD0A040DD8B2B6AE34B2FF513271E1B348DFCC563CAC1725040FB4BBCAEEB06AD9289C3FA00647F9F824CABBFAE1AF77D0BCBFE706E870536A953A208B2DF4F5B9373D853913DDD9CD357C68EAA2A5F5B0B3EE66802045975123E196AFAE6A4E97CA81FC0EC6EBF71634E8A3EBE07C0D0DD1B6808970AC17B6A16909A7BA222D4BA83C7E5E92BDDAA107C6A85E0C8F51D4D4C244A4932C8D10A7884FBEC0E830CA62739D48E29AB9DBEE4DB05B9A22CC3595FB5AC490108C71D63D70328FB073BF774B1B90CC75732239ECCED6E999D5D9BE7DEEABC895D0C183F6D4F2C8AD9DDDAFA07FD460C6474D3C90A1960159AA7086933053A16564244E3F2EA767BC77983253C6B6CC3240C9A5971F1C4F28B885035F6A36775E858D3AFBA13A1DAFC6620D8D86F607BB539F978F6028C7D77896EA26B05CA2E5812D3145E33616481FD3BA2C3C325A9C55FB0A5C8843339027E5B13EE7819C0645CEA2BE4912BB6C889B9EC3D12110B8A5783500F2BBC9C10DD1F203C6633CDDCE21A26C797ADADEB05FA7418562FBDDE4D05E34BA62DB0CE78F6AAA355D3B79B3329851D65C833AA0320D0D5846F4FCAA9E8A2AEC5E2151F87A91E25E8F477FABD7A588673CCA54078E0F56058C9B3C13C2F71CC8AE52E225560385D652BEB776F78483F0A5CE05D50D2F27B37E3F7A1FDC00E2C0BD554953903D1FAE2BA1FBD8DD07EBF85D52741561D90605EEBC8C775AFC7870C3C3F4BD5CFBD8C30C0A7292F22B40B2CF68F9E5C000ED731A85ED8D8032041DD92791DBC434CBF41C66038ABA11E855666B4E21C5A3FDF1C4E2FC0AA7F5C088AA43BEBEFFAC40F8F0819847485FC663ADB09A4B65ECF77A4D7485E00DAF97A80CAB31A101C6F8909BED03FE74AADB02BC6231CE4FAFAA9995850723EC737AB6E29A5BE2DDA1DE7B3DEE75E2347F6FE8F15146981F02D6C2DF7F1AB164819EF14A54D17E8E468C119B2C85E0CD782403BAAB1A37C265B938841D5206BB7331B34698FE9B819C76B0668E5F802B0D6D6E19583F437D36A19FC74657F692401168CF93D5CA56CE7999E315E67AE7B69C39005284B6F3E7E89953516678E5AFCFD78F3758FD3D879F7C4F572A98FAF0E805FD09E3CB01C66032B05AA6ABBA9B137CC08D3F23D90BB80CA882BA3E2539B1B39CBA12D858DCEF15B1915A0BA2A12F46A435BBAD82B85595C9BEFC75F697852729FEC43302768AEB22610E7EC9A924181CE5516D2F39EF4E16F1D8B89B131C89CDB0BD6DFC311B0301134EDCECA679E21A21ABCB5170DDB55E833518C025F47DE57A21CCA4606F23468F43E8734348459F9119859942814FC83D9D2FCABD9EE402C4E339DD3609A164B886605584D644773FD6B2D53316C8A12CD6E1B224FC4C4041E5DE371F471001658D4F477A8CCB2BD8511CA9FB0204D8D787B7A8FB22DE749648CC5015B0D987678242677D99936EA70F258F4EF0D079FCB1BDDEF587FD007D6F98A5D612EB121ECA5DD582FD0E57535973B395670255FE46B47062AE919A3923DCAE7EC061D1A28F76FCE3912EC74069F7C0D0291B130A94B7E9D14D88FF9D23934ABC9F589AB1E9EA45A5C1570EBCBD0667280B357E48A14B6ED53B8F2C62AE289FF4F850869C330BC15983110D4349A6A88D72D02511E654722A3AA6775188E3DC765", + "k": "5E163575FA649D21A3CA7DD658F2D98E3C44723EAFB2CC1A2D448DC9A518C008" + } + ], + "encapsulationKeyCheck": [ + { + "tcId": 156, + "testPassed": true, + "reason": "valid encapsulation key", + "ek": "521CA0F404B0DC436D02EC4D4C783A9FE60A027716754CC83A478EF9B292C71B1B4B78038B582944AA857FEC42ECA10F1D42CF97C6790F4B26152C5328279CC5A0AC9C369DFEB82A79E498398B54357C5B6B7AA3935A5BF456865C93669E0721ACC1C1233BC75ADB9A2845BEFAD66045D93A9A1A22024688DA726B38BB84D8C246A7E67F909A5329715212EA2F8D3A4631645887E0A32DB14957A39DBDB550395352944B7D86368BD4B41F11BBABDA9360B5F7160D8A123439B6E327CE6A55BDF08ACDFA1B71C382ABCFE9209B8C960CD95C7EE8B1A9A98CAFA4242FF25D957A82EFAA926E99372203381F801E0FA1898855A1A228C533F295E342122574936CE4925DB87EEEC11ED7306B0340A44B37C1C819ACDC474ADC004FC6CB2C578A2B089CC4894BC53BC4830F3B54D73AA2C10CA7E46C322AF962E3AA6E45287538B920EDFA71356B0BAD9A0D1E0855701CA4C0E23F66AB3C7A4615A5076848D564A9A154A00958CDC9152D61A9915484E65154C4D33BF29285A2632AE16C81040B2C3618718F807FA0821DA33A7B165379EC3757ECC21FB9A08DDAD77A125476E6266D42E20EFBD99488192DD735A196A5B2D310CD10E3CF711B441C642887CBCF5133B056EB8A1E5089540BA2E4404F25B71FD787BACC2BCA73E0B9DE76CEE2664AECF5499A687C7C167F08836A15B832E69B66A1509725720543A123311159063137F25711250016AE983684112A0E5B0AC03AC28468606ACC5CDE2450AF086E386149C76015CA6ABC228948D8029B224B63A0A485518753E9271B74DB35EB080B839A174326A57E332C24F3506EE748D1C56835799EECE13C16E53D3A3966A17548526B0D9132C22788687C8BCF3288784B31539D615344ABCE04258B6F218E48A78A4732B91F75455024ACD237A3B2F143E447533786889693AC3B96492D58765ECA406317085DC659BF893B7C11CE4655B2452A389CD912BB98A63DFA37DFBB71BD58A5CD6A6094114101940CAB2B90EF215A0A356098AB06EF068002E165BC21ABAA68B58891445D434139746CB54402972B12DC75B53C471ED6951498DA464CC4BFA76B2C5E8165D6D1CC76C3506261052E411B1DB34BDB6CBBC4A8CB0EFB980691AF6508CA8AE97518EA0E689747D2CA89FDD44F017C49FDCC6DFB10465C8412B175697F052940909BE126C5D35C0E642B6D251B7D526C11407962DEDC06C8DC138DE5233C5C976265A8DA7B3683C4467497592E8BCC741449AAAC41050BAD867C36872C2119A4C7DB34067A943F0C951168252EA0FB5FC6D769F940BFFDE7C46DAB183E161E02046CC708CE817046EA8063F469148D2336432B36EFC266077AA88C85A26606AAF7EB5AB6F3B64ED95FAC5C3C6DA380D0DC93A30C437A14816C25499B29132D492C3792C1880865B3670857B1758A97B14A29B569844775444C915707E2870B79CA8695759719A34D73062F51560FFF19684F1048FF9730E2669443EA8635F303214A629B71A04E6042B06584E6386F3B396304535BB8356240353EC8EA8158C01565154BAEA1509DE657625C2731A01BF741C5242BA7B909BDD6D47175681B92898AC8872F013852546A982081895729A369F40A70073B7A59BFEE3460982307BDB163D3A05664A91961B12EB4248C190B8D980392BA0ACCD9705B4D3BA54535426F10099FA8198537ADB66656B2D2843BE15F42B93FB4E521280031B27798AFAB68EA084EC0767FB5C76F3A129D15E985AA18C899A9532DF21D3366024D9C214B338DA5891BCAF287C4940E5B3837F66A4AEFD3C4A8406532FB8FB1CB8E45691BC1E960C976501FD532AB164D20A3368B0780C71A0E68C76A59F6CFC7F6663798B102C0A9FFA90C18E33121D127F6467E49471681B186A752B63B593C02F85F18384BB8F6087E59AE3EF879178534DE3361AC4A03448486BAFC394D369F450C234208377833A5B9768E13EB4C855C7893020A8091A8A9709A13693CF44CB842A747274273989C28FE3CC0753573BFFC565BA3BE02D7348FE98CC9BC616AFAC99308805ED0A1480888168790F4FB5417F27BB7F96EC963611D2CB468A98C508140CF5267C90A21E07BC471064EFAA4476D0B95C9A16EBDC34965B3B219DC759AF65392CA3CD486A2B9744AAF5BB33CB727E5BBA176CC4FC28398CD1EF5EEE6429066EDF3FBA01694C6AFF0F246418712A4BE53E1C13A75" + }, + { + "tcId": 159, + "testPassed": false, + "reason": "noisy linear system values too large", + "ek": "020D58896B0BC35C2D70DA8992C424899A8BA9414A1A8319A70659FCC6A3F2B794E84B753B0C8822ABCE063106F4A8706358A0A422AF8FF287EAC05A3D8AA84339996763A8F7A9833D2980F19754D18C37E8C7B46425BF8CC1A75CBB7E1DF5AC70E2056AD37F79F4AB83F87E5E32AA2E36CCA999AA9A4C82E81CC94B4AB471785661349F0FD832297A3CAB38CEFDB45FB020495AD1BC1108186BF4B31D8354422B21AE719809762132E076E5A5B692F40C704ACF1D3904D690AF79F217EA38A8545281F3309CD6EC565A5921FDEC938853A63EBA0A1E8A68187CB5A3D639D35439CE5A80985BBB9C7A9762EC14D3EB5501380D7C298F9FA177EB5A510B04D0CA7A19C763ABE4494A72A5BFC985BC403739DD8A9A13681DD39BCC93E965E4A09E44B97690C36D0D58B4F02126E4A0A07E6AC10E58C24B554ED89AAEB15C49FF53C16433835DD72FB4F69458D1BC99330891F1BE9AB172242412FF819F82377914B269B7120137375BE39B9D32112326EA5CFB5A071B479A73E0532B643B53E58ACF1A987473A3AB017A66DA9AC5946097679BBADAB1A0827783E07F7BC198903215A6777FA3B77DC264CE37636AAA48A5F4208C03636E71C7746083BB7D403035390DD78118FDFC2F7DF282287C96AFE8774A20CAA6DB69104038EDA590C648C69511811E9C28D52C3540721428336EE3D7732A287E51109BA8589227864F5C27C5DD07556B33CB4842340E5C5B027464A74734F088A591EBBC3CAA0899359E75C395A6A57670B04BB38619BBB82A27C85F063CB286160CA46C372604B794D4202EA16F8A757E9DD8A54194C3384850D58213C7533CADA33DD6704F082B4EDFA67FAFA2CAB0C89897FC4C8F2255B839CD653C4F03A728F0D25CF2B99AFDD0419C515DBDBCCA0FC6947B0A0C936AB1F2F290DCE5474183AD4DC85218EA13AFC5B671926842A15D7C367D04EA97890B371297491D21720DE148F9780866E1414EC6844EDA97AE951AFB0AB07FE61B1B856209DB6DD522792C03332645A63C460EC595AC5C49B1DE1980DDE8B4BD5505FFA7CC4749C42F0694BF3ABE4CC6ACF8A7235766513319915B58522487C7F9EB9C2754863A40036E28332265C2A4FC420D95AF87063C244233D5C27CA5B0261669386B624B5AFC992CAB773EC948B8957079B3B3FD27C16CF969005B27B4C438B7269FE317024814AE56862D18362FA47A0035339A485528540BCD27A288489C80AB890BC3992BE39249E507CCC92A250EE631297AB19E6C39AC446EA5D54C675368BD5BC4002463D8CB9A294C3E427A2940BA5F00B97FB2B1714CB6880F5B6A13F868FB936E23FC4B7EB563E2F1C5FB421A449715A1F8CA0D3C6AF726CAA5D93C255BBCB7E60843A54EBA470B32D19E3088C17AE5BEC5408ED9758639C2AE50FCC6C5471B638210F8BB1509691B312082DDA85C91B31517D78D4CF96CD94335494A66B8EC3F9B2478DC264A1B7462DEEA1C44CC0D357C00604ABCC2437404771640D01FA1A9A73E900EB0B7CC8CB3A5EF621EA0118796AC7907612F339654CB519B50A04B6C38A1AA7464E34A8E9B9029A4C1687149B0959070CCF4744A03951B4AB9F6B88D85487E2929338CC6CAC39651EF7C9B802657938B7C368C2B6B0539EC0C152EC95AB702662C21685748A968C68C150A96A9050BF3C597ADE0699930C9780287D8811A68A74AEEC8C335D4666C64AEB257C508A45E32D7A6CF86448B81486296CFEE29C3CA7995FB9B55A8D62C6BD3BE1C267627F4754DFCC1A7818051BB0448219794B87089B8969C72B01FFC168703CD11992383552E14197AB1C99AAF848231579EDDC305D4942B27957700ED5AD44797E0B5953AE06EB25CB0CDC94C6B67A423CC6700D1604E362FB35105A8E6A00ED6587A759B98E37BD358C9FFCC4465F9C5D945A93954550A2B45A6CA90A3593E07163D7A8AADA9848924B135C4911224B52FB5C2823F499F98D0238811B86B55A0D444A2B14B885A0B23F510B5B80280830B5DC705122595578FE076FF859B5FE0878E7B17932A3B99AC58D9A86812AB2006073B8D80BB940931CCFBA269D319D68CC6B1FB5F6EBC288AD208379364EFA73384A8A441C1759BCA9D90016024EB2316D50E8042CB74F81320D089F6D08008007B7031A02BF773BB1C620A73B3A1579FFDBCAEF6652400008B53F560F13D5DCC0B716895116434F2EE28" + }, + { + "tcId": 160, + "testPassed": false, + "reason": "noisy linear system values too large", + "ek": "02BDB4EA986EA8C4815085C9DF937348E86BF9CBA6BB910F2BB760014B2FA93C86280699AF5323D0D2C1DE689F11139BDC58CCC49771D5B1B2F17C5FDD1662EB0C663CC4935DE26E64DCBD855BBDD42CA9F37C9263575C6008AEA8803831124156F71F22743DD257BAD514C49DF4A4C2828E0070130EC1C2026213AD9694CDDA308CD82F0C437557251807392508067D3E3A044AB0665B4A23AFA209128A010C8C91D685BA77C65F119046D14A196C98882D135B14303F7852B1E4DAA8FA873DD2D889186941266B744EECAF47EB84E47A69C86C16C71279DDB8B3DBC7A52B7B2D8D06AAB4F65992E79D82368C6C8A7EDFF47F05E00D2CB35DCB2653FBF51B7A695721C68C602B6B08BBB52AD713480151DD221DBBB129F6D30521ECBECAA98E847470517968FE34CB22EC2245CB0EA0F0B1A4909406EB864990AD7948884E1562F3F4AA823A00553832DBA5B5E15027184133B446BAEA9216CDC3C633481F6AFB0263965F2B66C01EB180564C7187572073C31A00979F591BB6516C0CEE373D117882434A8024E316D1D192EF8AB1DC1013AD07A8E78C226A3884C2A1B8CA372499E784FA10CF9F98A1C22A2F47EB4121B7157AC0C2DA66350FC961367B92F57CAE774443F705115C21793EFB365BCC3DB7D19F18119ADD93620014A69395C31B73A3709685B113B576711C6756132687840F2541BDF6A70A0704E4B4C2DCC33CDC9134586A50AF2010FB640A3D03411A0B8D96E394D27733E7B58C521184BC8AC74592BA596220A1D00AD47240E4593C438B94EF29356D65415EEA24C11644E5C726991A5718A9B21A91CECD83AA172C9495AA1B059845C81550DACB195AE6BB25F32F2CB7928DC1B83FE32700C78C542641573B241C92ACE3766522A84669808DC6F94F98107C3CD15C1830AFE473008594BFD1C779E0017AB9CBBAD49344FCFC6C38B9A5F11956316CC34836768CA99D8C48BDCAE189AB93A556493530CC4E8775ACE118B4095C353C9702AB023928E16981E61F325921605659825034757564A494B8955759DCA7B10593AFDAF396DA7B24FA9570E516BDC2EBBC36D875917224AF4059F6863B511BCC5B186968DB28F122415189B103F4B7AE003E87998244B71326CBB19C34017E892D2676A018268A29A77E29A2697C0655E2455AA544844939974E961F204994DDE56573158C6120A2E61B56D7663926702A1DC77BE18957D789A512632852F60FC6220789CA9FB2868E49F95903C62E0CA13979F582DAB615047B06392BB0DD9B6555D934E9E18BC1987CCA01A0917C82DE6029B593994A195BB501A43EA689FD8B833BDA0F9DB2890C69846F625A84D2C97F47A21E345774DCC8C58AB7AE34BDD838595CF182DFD837EBFC67F4B96F2FB2C2CE046FE62B5FC4291C26B63F258606294101BC5588D13482FC7A88645111B9FC195A8C15FCE93AFD7B1B173C987C282B879A4651939F83D2AF413BBE3F6027299CA09A8838EC79700DA19E92FCC717D69ABB775BED5C19BC9798D6C32AE6279AF473356F5B7994676291E00A2333766C5342FBB2C7B261C3A10B147C8ABF768B4781495C434199A4DA7BC18B2BFFA1268AB745CE972D5265A9735CBB84D67B7B236C3213058680A69C99006CD0850035414B23A40B6670542A98335664EF513700DA6A48EA8290370F5974B868BB0AA7B3A145A77B83A01DEF743404385749B177B5259474F28377AB811D2075F0E49021F184D1B866F3A97BAAEA7E54A292C353AB8C974D29B02ECE23835B163A120CB990D81310085A6A18BC6D69596BAA0C1142B297C18B7B09A1D486A48A812EB5C2875B71AD2D4CBDF8A6901C3C64A8472730B963E7DA999269A849B376CC6947E3218326B96CE175BDB7E79CB1A20FD8439F276C7ACDC47C0E266AC8F270CB349CAFB05D2C02BFCF5321DC3A3320506100E27576D58F0A0202EB2AC800E0801CFC81DC0AC680B3BF25CA9C3B5A40B852A547999637619063828E1C6C0A6FC73B61B011FBA9101944CE179A504BF6A5434948628A0F0A35279480A287E9146690BDDE5A41AAF5148351A3A7C537007100C4764C18BB0C8480C7B34414E833257F1434733896CAA22135DB24C7D2B3ADD53B5A110DE4A5A883B591E029462537657C73A24FB92D6DA735E6035372758299963CE1C91FC62EC34416C0CE6482495C96BD8298D5AD3D88F9738C005A1198ABACE6" + } + ], + "decapsulationKeyCheck": [ + { + "tcId": 146, + "testPassed": false, + "reason": "modified H", + "dk": "B94C8F189B7CEC12B6981C64BE0509E6A2C6F687445732862C984690FACF0AD6817C863F4AA45F9D87C0ED432EB1318354A60BA8EA823E9838FC1B08A9D7C5569976BC624378882CAD589586C95B27C6448819AD8F59A10B60B645BA69C9479F604747D2DA20827B6255AC01CDEB71F5B157729872CCB5A1DDDC715D0768B104AFBF279024366E92954FF45A06DE32C87818033267611B2B9F20A2246DF979FDE2C595929910803A2EB8B4193B029FD51594015A2CB48DEDE90D8913B118F50C9CE4AD1A6B3FDA8B6EECE6673EFB3F7B229CAC7B5B23E86ABCBC2E58E5CACE626FFE040D2EC83A4D457EE014B780727474E51B93A71FEF1545D3C913384A6375E62AC54404809C2BA3245678014D636A435AE50FEB45124AF16518851F2B121FBB982170E5AB454B53AD8A9D3802A400D13A8A724A8A1B9F6F49C30D059695D520BA73A370FAB8F042A3714AC400714F66328E0CD473E15B71AA62773637BDC4B29BBA1A07EE15BC73839B7F08B9EBD4CB36D2427E3CCC92266F19EBA7B6737FCC58848BF0AF174613A4316D7933651FB39625496C26CA9705E48BA105046E8A226DB181543A0C26B455482618460680F522C92F3549F0B87A14915D0FF15C6BA5862C556122311A4DDB07B5248C3ED995F4AC80D015B3E6A71C97226BA61C189A460D6D368F587825EC51C50669269B8B8872B9933FC1BAB5472D90AC8D36E453242826F7A7319FC8C262795947C2810104B98023AC84E854BA59A3319C6EE6905E36D0C3B0B41DFB393CE378046FE292097C528EBB72B91B20C637C531D393BFC43D54692AE4D281E9E637137463A0701D286AA89ED464499A1458913C586A2FAB90506B8097E061C0B710B68EBCACD8353C7CCC0AD37A0EEF32A4CD1232B69AAFCE9AA5620B953BE52EC7594D68D57976EB1B38CC677885840A463C72C3CF17445AE4202AEFFA4A7EE0275134162C4BC3A82BAC03864D39C9406DF72EE60218D83C0E0EDC067A1592D93A8FA32B84CA85BC51D15BFFBA17F7D740143A068879362DE00669A0CF7EF70C604299DC3CB52F023E56F544E3E35CFC9739A64A9392C0AD1D927D62F09D96418F3D926E227C26E02CB1E2C10B89E373A1D515D39C18F259212E5C14A7678A3A761F18E488619C46D2A21590270FE511ACB7C3BB45F9C699E63596DA08827458142791893070D6A130182430665A3152F62328DC4C2294C874F293331604298675A4C2C31F79ADEF67C85329082DDBB222DA4DD0C11B6FD8C9234ACAF4D40679C82BD0479EEA17CCFFD209409C5C2AF148A87C006B65B39436BA5CC6440812ABF7B08364764BEA966982979EB55684D1433E71A471FFF73AD423554514C144CC4377BA07197CA72EA6CC69E215488CB91C7980EF74CBB5E780931A34CDE964DBF21E3BC266556281897530361BB3F8B82CB5415FFFA1CF8E32B12E2124DCE774D6D692E0F803CA357536101CA5944D4E1A063679B0937ABE6BA20A8CCC75FDC03AD4BA2F61522CB8F565B05232E2FC104AE09EA156898E8B2D3D0A7FFBF195E9D1A623E73AF87181B4D932AB259B7D2B3814E721D499950D184E56F109CB1106E9E1062888B6D3185DC30AAA952946C81B6FB83B507773877F7C84EF142C235241E2901D72CC8A6F481A19721A9E877BE36369B4F614E598698083906061074D6055DF372510C543458CCBD9F10881D209176396EDDCB8118B3E5A22A989A56C67912721B290868A75F6D820802A0FCBF00AEE135139D8A1868745FDC22D06F05748D323875187FF298E2D37240AD93EF7A8A3CB462665209DE6F1990828830A65446E2A4C30958BE3AA2D4F192D673A8E99497084544122A56BE4F6BF6BD72E98894E5B696A6A19CCA2C1C79FA976384C3670B8465C2ACBDB46C1B5881F5BFA3480675F83F5535606CB393934A6DBA30469282A3127B5B77BD7709350091FFF453CF8F76370F847ABA2B78131093FE4B699D342BE220E085458E150B081B81FCDD35CDFAC605FBC4F9757C24D7AC542E90EA4E9255B5B29F78479AB36AF1BD691D005AB401269BC9395C9986ED9A2017E3818B1F87DF9846F1EA1A18EB681A0E561F9F379214A6CE44334CDC5C4D489436A5850B272499D32165D10CC8B6577965C56BB55034181BCD7C81488BB9FB1DC8D13C1536870BFD4868F136CBC490B62C79741B273ABA970742D7012AA2165C297A844AB4C27414360B5CDF0E29D31B5CD802CA3B8149B00DA23F7E213A736017C2B9D8846C12E4A5DB12B57E83168D8D6BEF82CBA020116758CB06BF694FA186000458BDA4358C537ADBFAC277FEA5C02E1C8B79998AF6C4F1A6CA47085022CA7A6CAAB0780254FD8C0CB78223D86B8656E56231E9A28F0612747E2075E25B602895FE7C498197A62541420CF839C31B347F478BCED325336BCAEBEE986D52C7AA5B5CA56723F1641CA3480A040D1267A407FE3E92D2C8023DE00846AD79B0AA89CC1F99965E88DBDE06B66415E36D2120E2400F435C3145B57EB28C62C248FEE401525FB3BE6491E02D4156EA38C819AC59D55CAAE14A7D1D3B9E2CB57B7BB58704A8C2D239E622A0D15A589916517768387E21408E0E81A01216A5D4A636686321B919E7019A1697B886E81A376391EDAD6C5E8044F6BAC63A84B02104221F19AA8E3468579A0A156897FEFA66A11A37FDF46088EA586AF8804A9854884597A7327A733315E020693DBC62BF4A33E1DEB74E91437EA30670E7430B7E1596611A10C3595C3AA14F6337B55CA5D0B953068D52EAEB7415BF18E88D7905945CCF3140A75263A25D931F802702EEB40A944BA6AF03FA2509ACFE753EC60CDB0A845CA5131EAD4401E2C8D0A32306FFCC7DD189FA367AB6A4A9A6DD1806BA70AA4440BC80B99D0C91C7931871B436F94B39B1D04448D18085C5AA946E642CE04143670BF7F8514A0797C6FA71BE4BB1416A0B3248075251CB1A5B79E36862AF4947C76D82CE8363B89139808C4CBD14B3C4F1700160B61E5648FB954C25009BACFA265F1A0564673AF000120D93A2D4E211CAACB1D69713D4E183268803163C859A71B92D9117FF807089F47940C57955E3343506762B56272AEE182C719A5478444F29513C75AB44E3C4F0658111473CAB33933C5E2999BD01C5DCB71F1F82F6CA57B9EB6376C388C20064944E3AA803574FD816C119A312140117BA492A3D3204541335C082CAD440F0B22BC55A65E2B852E3EB1042ED046E8A9B755773425C920D2F4260CB29D1DB76A879B36283BCFA8598F799302DA5C14C5AC03230A338EE3C8497899A92CBE645B3D9E91499F441FAA5211D87162E76262BD795409559B294A0453F028A3C737962AA6B9969D8B45A934C7CF22FC2EF51C661541202085554BBB5AA2B207D161A75B7831000B8A532C76C1D01FF3A359212BBF79137B42839A217B009702192A92470F4B3DA4A9397E51CEB6F883C5168DF122076CA783C0C2BBBF572680E5268CFBA41E8024EC58A133A8934C46553302753F39AE7E4B9311A984CDC68360293D94D8A6D857318D5B257291685C5068311839A489485C64AFB7430376108502279FD47042B7216E8F7616EBB408CE4C3876548AFDA6B31596C21EF096B92782D5D745885106C3770E1E04C1DA2A027912137CBA3FFD2632B808AD0BFA8F7D2C9AB4E24245C8131AC5167684190583AD527A25BD8369FD747F29528C61287EEED97D6CCBBF791AC8A90AB9A5E19F57482A89E3A9DE9C00CCBB9884C80F7CC7680B654FE1A3CA5A160431A1C42CA445AF7A4D60CCA7F6A27A7B919C6D70B030025D86341943897296D61B0C8414A696C95F70337C54BDD04C08A3FBC5F9835B9A080496A1308E03A88E76973C85A9DA0C0340B95C29228DE1FB3D4CD96A9204848A92A3C912B0BD8817F6212242D06CFE803E91B656661A51A6A1841DB00F1F8A02518A971F61199F93712C306CB944C52B469B04E76389E71543896A410979009A2A9F09CC58C0868294A72EA8A0FF96A43F543E5E96AA1934545F755382859F962615B8B546D5609B59157DB4CA4D5FB87366E03ED93A2D3E223590A3B9F03C95327A35D65C5B957546D3F0745A31A3AA012719340F13992D8BACB18153A83D59295BC497EFC5C875C4542B430285F54FE245558538B15CFC03B69889E2EA1B1E11CA89D165C9ECC1278C2D5E878C82C13D5AFB7C2DEA9D71437924D97D698180C0B626E683A2AD3BCA5652C9E3021ADF322BBDCB6D9C762D0A12270505A811E3C27FB16A9493047755163DF828E0C8B0CAB4B32EF17C784B0BC1A1922967112122792B20A2917A7FC46016556611416675AB9C5AFF696F80D48B659ACCE23FF815C80547A9762CC0D85C3F2501DD291BA56692474A2E13B3EEB40A73D7BA5E222A09506341F00EAA4E856209AC3525E24569BEA949B03B755A36972911F56028D4EA3F21B1537B186287A191FA400689D618C2C2C20CC091EF65" + }, + { + "tcId": 148, + "testPassed": true, + "reason": "valid decapsulation key", + "dk": "31C48008D52C5365473A1512E7813452784A12410BAA3313EC0C0B4769C6E52CC31776074B2282E035B611AA1AC8DA6DFE63C0A1941B473644162288E53A5EC9C59037999DB3E8C6C7857CA57C42C7501182740E7C041A7404620A9C76F0F611D010703E720CB1004C20E27274B44C0CC3967C8566BB07442A40CD670C0C6869AECD22A57DA32059C19256CABF02F16AF25A3E2C876CB575C12B13871A37B61A00176885B08B158C6B86C02A15539DC264A951C9FAFB2F26DC1C65F3040EAABBA5186A2315944E4B7AE93489E896BCF8F677FB7AA24A180BFACCC19F80678EFC64F8638280B64E1FCBBC4E2672475C5B4F4689E2E6A0B16B262214BE03138364341E774466C5F52C14A5645CDBC5691C9C4D7329C596089EA8B10128BC01E8AB31A56708CA7185FAC4E8E68EF8B8096E6351E19AB2FB2A7E0EB35282198B8BCB3783C05F71B65FE7F389AC166D3E09AB767A14B3575153801A7B38C3F1B085FB28777D326954A66FDE72CCAD43C92E971E2365563F987669E84CDFD43EF9312C5458BEDF9C5EDC757F01365144BA2F4DB954FB2A6168B46F534A0277CC00390B81AB07164A14A830B69ADDB5011D37BE8F7332CBB5AC3E1C042E4373E56B6180459785F08F6311405348990FE907C6F9213321CE23591ECA58180384CEB5B876FC46BE59C559F77AAC21D22BF384901430CB5FF24D4CC2B4AAA0B9A95A4F2EFA60D396040D4315C83911220947A9E0CA7D2C081CC6810370602FE38B1AA9711A87B4331B7F0D4576D1F20A16F0B6EDE5767356419739777C5380FA0411C1052581F412E0C050AB9393D997BEA1E51AE6E32539795E66C46D9C5A04A5B56A8EF808D8AB8CA7D82609E727CC1029FD8002E4E551A7CB05DB27841BDC3B634922B1000D4395AD76039A80745EE72A4A6FA12BEDE61A35D58AE7752B3924CAD762351E598C10354574532583C07B82B4BFACF88C18D12674EB1F3F28CF25DA43BB6541CA6691579651828372C90067E379A982E623BFD062087138D31957F05136ADE1173E0390A62B07EC061918E34089543B9D1C4170F40B1A562C6A11402CA00A3715A41697BF5D8C2E570C5F6AC89D04698F9241CD135C2FA69771A3B9CF1BB793D1E6C945C97C7665658753C31A00778F72C89E30BC2AF746AD23AC21B40E299B57B5D44E55335EDC0A3D1EA42D8C7855166B4CD0229F6CB2A4F9435F412137AA904BCD0ACCEA09B8ED10C9922594BD756E70873B1AC52A575C1210D0A4CD80AB11B7200BC373F6C208D725605207C04D8492F8C9C5E6C1CFF023AE8BA547BE1743582044E5D0C4F1F20534A0A62717AD60D91A6C7A03BCE2721B7503E75581A23BBD8F1848C00AB5509466F144B8FFD5C5EC1370EC8B7C960755A645C4CF23BF4CE9A19E3134D2E5830706910DA7B0CD9AB92D145F2591CA2DC982E6E66716B7754A82BB335709A5249F88BAA71C895BFF403340CBCF70768CAA563F5E9348F4B3C6F29B4C77141FC5D52AA532287876590B15AAFDA8289B9C8FD4EBB0EF8944BB99A0CF83661D78727854B0950A29715A4AA8732B65A08E6BC81F123358CBC3C2E4E93FD2307895597D6D66B16ED139E8B2B260B63E14720BDEA963F8B537A85A1A455BB176D310F9855F6AB019A4D919EB1453C8CA1028260C64513584108DF2D56B6451172B674270744328CACE2E167359FB594106630C887973B35776EC50FD030641312A623908DEC6C81FA573D13C475981ABC5D30F77D4B74999209F4A565C1233E3CB702897379D39B8B68A43C4199D7F64889281BCDFA47ECDB1312D981CB9AB8AAD764AB6EB88216CAE97D93D75E3CF1E654A21209246023B42376344D909C651BA92A4A9CF1CB03A522632B41EC27C8F180CD05A230A2F60A12110C8D832372EF1A1A3BA1769FB49888728BD080DC2E476971756A4E8AAD5C18AA807D0D708714CAB2BF4272C458C20DAE8B089A60945B4B5C5697F31BC4C4DFB1494830323D962C8520A6F7C6C11F8613BE83C22B362A352B47B2B0838392A2DC4745068211E238DC385211BF048ECB2478946C9C2B837CBD0604FB241FDC0445008013DA6480B826EF484AF939C4A399101D380CB55C17E64B081C201D016261407A6C95C570E7BB08C79BA4BBBB92260388342789E7D7784A0E80E602A7EFD97CF79202F38E8A1C0595DD59C97B7B559D344517FD472C8B673FE5012CA935E6DD59E70614ED7C86244132F8771AD2D1A9A884570F40AB82317BBFCC5879844A6C0C757236C94E8D6923906C065B675B54053176B4C6AD2AAB8A5AEC323CCD68A5AF8DB0AE94BB4D431CA1EF198851BBFF64B4EF96C6D0EA045F82C9B7D48337D2A3F1EA7185C2881A4E267DFF8407748C03FCAC286F302A16173128396D76A9ACB217F29353AA87001EA9663D8F48B1E4B0D9AD6CBF81A6B600B763B75493577B85694AF8FA2729323CF5341961F423770217F34466A9115929447AECFD741E39C6796C5A132E831AF3CC2B70A434524C764C9C783EA1D4F269BA091575139C092D91BC46C4402CA0C65B29588B8A34E7C5744FB38189279A507CC9A810588F8166F95833183618AE2470BB25AE12B330AECB4F296B11E8C0EE2E34F12E2ACD0C891CC136D033A17E218A7FFAA7D96807267B083C1CAA8E1D162283AAFD98C5AC1657883CCCDA04A9AC8428B5EA6BFA4B04336D6A6D222B79714CC9696C8CD62B286C04F7AF69F0FDB7ADFC999DA613BC8F6A156161C2E135E7656938EF7C80489B0DFB7AEE086C9E66446EF1173569A8A7E23A0B82B92BC50250CF577236199322B783D6B8C5AB82C9C378716B231C4850ACFCC131F03CA51055A0030CFDCA874CA08BB6E3462FE517DD5F282EC0C5DAEC64414C4534C264D68DB5C2085CA25644059B2AFFCA64BB1D8762A293D42B79E4A9BAAE8D73E1012C114702D2BD19629D646B5300B0500167F3C9B1A7145FCD95BC712BBEC42731D321D7DC8C4C361C271DB1863759B147BA14A524690625823DB8772CA0E2A45AB7C734A1FF0951156A305D70B458AB422A6557AB423641266464A88799987B72572E60A79D0732C84049683310EE475B695556C4A65615651734F56A90D25178D604CD058230E3338FA94003FFC48AE600D2660212D648FA8062B38182006030857882B26E62BF8A23CD5D275B5517F4EA736A298402F155B2141A3AA8A74E5F69678185EB4334FBC380980E850A1399AFA0C4EDB172EFE326BCAFAA3A0DCC07575455C084442E2B5D0105C39B234B409A5BDA7C33C2B5D2994B7CFB90CFA23BA2A65C28E255F0BA3728671AB9E410642D1812D0969C88C721D867E36B8B4808CBD1898B05B563FFC23A306136100975AED0A0DBC1AA8E823C7FC402821F65E31972C3FDCCC96F55B16C3AD48BC4A9CAA05B4C0CB7E863A47296D3C727D7B3B90491364430CCC4756A2203150958770FE354E0F8803D1A64E75A309F7EA01260345AA1C3E5FFB863BAA4B0E5768F20B7D906766F8B86ADD6066CC485FA5C9141D16039E5562E433B511092C1D18C50F7186642152C6986100BA728C136DC28B237E2C8347B28E3FB07C0AE48A42F955476340BD5CBC145860CA6449C597920575AC49C0317F2926BEB2BA7396BD73864B86490E1EBC8A03475A9FF082A7F967F509C8F8BB0BFD4C98174B0B36D06CAC5789919C02B3C0162AD4C4DF639207416BBB5071E5F692668B5D40AC2131988535C68E68C227810C0B68E1B72103B61D98A32A7A05AFE1BB737627955A2276921703A5B0FDE38E415C4E752784FCBBC432A53BC4FC0DB52A46C3D0C3014041C870C00041C183D21D5BF277ADBCAA73D6241CEB018E5550A8E24FA5CAC9310B6735EB5D245239F35B06E803697186680B6AA8BA1730B4CBA23E105565F37B2B16070C3B0E77A965CF577F3C3A82A5192C4C3BB52373C54034B872604ED4A833F9E18E389A4BCCCC6C5991CB49F097771CABD8AA6D53020875080E91D198F93C2EFD276C86586A9E53AC1AA119851C06BE12127AAC3D662AB5325B729F38CAC505A3D0E135CD9A38A8069A7A2A76C5B39B30C7AAACD666EB061093B5C41E720B580A2914552AF509AE2035329D28050C5016AA06AE5EC62E17D84C9024494049816AB32DCF204015A092AEF05857794861175B19A72C9687B6E9EA385963BE3E6A197627359728BF47D4CEDA32779EC51B2F6BC8287241C7B39474805B5AD13DB6C47627AA40F91CCB6CD062DDC58FEA6A22EC98A65F73A0054B3E211ACA6904BC01A9A04B042AC128910B793448A98D19EC1A48DCC451B00F66629AF4DBC2C24101F9480C390226F7B184FF307BBA63B6A1D5B1FA4281AF4C358052CDFDE37B07B2C7627429859EFC72C31BFCA310E4A63E25B10BD4A57CA6A2611E141A8C0CFDE7A700EA3B849017A99F79273DC7FC7702D269197D22112996A9AAAD6803C7CF2AC8132D84A9B5016EFD34D64543E136993D98D12FDA" + } + ] + }, + "cctv_modulus": { + "base_ek": "36c99c6874b42683550244a579ca3bdb4bac1658603bc0c8afd035d0a5415e2c4310d647b6a3984244c3e7456af0e180d453240477461f49cdc6111e47cabf82fbaa36880c869262103a4a20cac30c304ce6990999039e7ef648d08253bc195c980071e61c959f02b409bcb737098af3c6acc98b579c99521d791295153b672a2d17d112d5e87d4878274e01568f9b81ccd50fa8ea34bbf15fc892c82ad46a21a60eef635aa89ca8c9b8cf944a5773ca01488b0ad35aa9ab5babe4c0cd63a90bc929bc3f9a0290851b79626229c54603b883120c0318c7113077aa4d507a366506c6936b56d1883f019918132f87592b5b6343e842323ab3713681b0de78ae607365de49267fa61d5b90ae69d378ab25626791a71dfb93ea06002cd8c32ef28d7e38587924363c1055554b4d7c632b144a958a9139a9ca61523a7e4cb38385f2547aa812c3dbb97ea49e7cd83815a486a6373e179b29a65a005df32b4f72bcb2a5c5371c2bcf8441bea92fe8c2183c6ca97b643480d8cd62d84fe17599f568af4204bd9a1bba594cb5668b3fc44a79397a052faa0569436c6515c5098a9bc25593cea3b694d82ac65516268b58a0f0a5cafb073cf943099b558f6c519a0b96fc4093dbf2362c237872160d7ca7b627606ad1c8169320285b1a3ddeb90d7806183b2bcce6228cb5a166904a6181c789041612af83b2c03326aaf8c468c6a2fa2ac2d8182da2941d7f34265ed54a0ac6b8f819c997a66c26d48c16086a9d77b25d80705da2bf198305ba3115a50ac655253044c4261e1b1d72876aae59b1e63787bf2ab0773c95d0707321ab91b5216d937506b4e3b3e49c18c1f1bbd0e4c1b10a5c2f13cf496769cc463f65e6cd26953692722218005bdcb25f165c9abbcc18b06b8050cb58550c319cf843c560583d0b499750231234c65b243055c68d16834d2a6baf14f96186a65d5de05eaa0b5a5af076d574a1da7334ee45489e337e147b297e963936fc084f3b7a8a324406422d3f677a4a5437965b5115730e949353ac04858d0a56af7ccf340093ee05309736799e8792018433bf55901574c341b2a9df1ccab2955624e58f0413074b1810f957bd202a728855ba1c123a0d3bc88855c59c8a8bafc6329db779b815c5d08297c8e0b328c08fa526aba4aab6c21b8c653c5b0b135b99ec24c11b9edccc4cc3d231efb7a045b61af546c24458ac876c151626cf3a0502f2463d850c9a038a200211abc7c085d29812e0395ebff3c236a34f44c64153e49a2ad002f1e16976b1c18766706fd3b7dde69d29834bee186f4bf4b7abe207175389d1b1b5c9c674e6facdd93158f17628e1ac1197b30902b3277366af0702a0576212cfd9707b47424971bfbb75011cba31b2a58ca062a479586aad4ca5890a8fd805151d5294df479803f4b80d055b19a1b7c483b280c12064213b2ab278eecca661ab635fbcc436fa8412861171f8cc49bc56cf1ba8050c07fc8c670886ae29d05762c26540d1c63888b7d6c9047cd2cbe8f9aa378114722b1d2ca87ce2599dce524ea6c4b83cd32133c36db8162e3c34681fda8ea2c7c9e80c87d34a73e4812aad40af218c75c2551cf86607d43c9f3706a9c30c737a89418ca703d2b1a015728ccb313e9d04117aeb8284f8012fab777640be1ef2592f533468851f8ca70087a83b68bb67cd9947334975e6355a9e50bc32418801458b3aab0d7b68542b581ee5e65990897830d37a72a6b849bb61e8c51cee56832793642f053a29f06bc1a616b912943deb1090a9cb671a904b503f45fa052f140cf0c557ee843c2d8818472c102a91cd7afc5afd9687a61765676508de1050a0fcb7bba82d0740063ac001eb336e81fb5a60c4a7143aaafaf93a62218e760b9dded8c44a6b4de9397d9c677badd9942f346f5b8022ac8c2a54c53856224bdae0146136636acc0b4815bd908b6bc525c0f06732034b8142c16490d419fc85873eeb56f2c002f1126bb3075f5bb16c99642e80f57b4df0479a564e50c78833485936a63c38d1cb8932c9050a0a7ad74a54f2a28788bae082c946da9e8d757afe1880b5d5747717698103538098b177386d36d7bfb96ca1a6b68d6ababc5a4531ea33cb053279b81c84c6d45bb1d14aa12258fc407de6a9a5dd66a9e16b6bd6b9b3bdb45a4a06d763455b4da200013e681c43ea7c943eb186edf29664c5dd42fb0f240f61", + "count": 1040, + "sha256_of_txt": "9c27ef8951a6b6b59b57dcc4e213ae73472c3db956f4298d6c9983c54bb2c263" + } +} diff --git a/src/test/mlkem_tests.cpp b/src/test/mlkem_tests.cpp new file mode 100644 index 0000000000..6e6a6edc7a --- /dev/null +++ b/src/test/mlkem_tests.cpp @@ -0,0 +1,618 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include // IWYU pragma: keep + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#include +#define MLKEM_TESTS_MSAN 1 +#endif +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mlkem { +static std::ostream& operator<<(std::ostream& os, Error error) +{ + switch (error) { + case Error::NONE: return os << "NONE"; + case Error::INVALID_PUBLIC_KEY: return os << "INVALID_PUBLIC_KEY"; + case Error::INVALID_SECRET_KEY: return os << "INVALID_SECRET_KEY"; + case Error::INTERNAL: return os << "INTERNAL"; + } + return os << "Error(" << int(error) << ")"; +} +} // namespace mlkem + +using namespace mlkem; + +namespace { + +constexpr int MLKEM_K{4}; +constexpr int MLKEM_N{256}; +constexpr uint16_t MLKEM_Q{3329}; + +UniValue ReadVectors() +{ + UniValue vectors; + BOOST_REQUIRE(vectors.read(json_tests::mlkem1024_vectors)); + return vectors; +} + +template +std::array FromHex(const UniValue& value) +{ + const std::vector bytes{ParseHex(value.get_str())}; + BOOST_REQUIRE_EQUAL(bytes.size(), N); + std::array out; + std::ranges::copy(bytes, out.begin()); + return out; +} + +DecapsulationKey DecapsulationKeyFromHex(const UniValue& value) +{ + DecapsulationKey dk; + std::ranges::copy(FromHex(value), dk.Bytes().begin()); + return dk; +} + +bool AllBytesAre(std::span bytes, uint8_t value) +{ + return std::ranges::all_of(bytes, [&](uint8_t b) { return b == value; }); +} + +/** Run `check` with the active backend, then again with portable C forced. */ +void ForEachBackend(const std::function& check) +{ + { + BOOST_TEST_CONTEXT("backend " << GetBackendNames().arith << "/" << GetBackendNames().keccak) { check(); } + } + ForcePortableForTesting portable; + BOOST_REQUIRE_EQUAL(GetBackendNames().arith, "portable"); + BOOST_TEST_CONTEXT("backend portable (forced)") { check(); } +} + +/** + * SHAKE256 truncated to 32 bytes, built on qbit's own Keccak-f[1600]: an + * independent reference for ML-KEM's J, which derives the implicit-rejection + * secret J(z || c). + */ +std::array Shake256To32(std::span input) +{ + constexpr size_t RATE{136}; + uint64_t state[25]{}; + const auto xor_byte{[&](size_t pos, uint8_t byte) { state[pos / 8] ^= uint64_t{byte} << (8 * (pos % 8)); }}; + while (input.size() >= RATE) { + for (size_t i{0}; i < RATE; ++i) xor_byte(i, input[i]); + KeccakF(state); + input = input.subspan(RATE); + } + for (size_t i{0}; i < input.size(); ++i) xor_byte(i, input[i]); + xor_byte(input.size(), 0x1f); + xor_byte(RATE - 1, 0x80); + KeccakF(state); + std::array out; + for (size_t i{0}; i < out.size(); ++i) out[i] = uint8_t(state[i / 8] >> (8 * (i % 8))); + return out; +} + +/** The implicit-rejection secret for ciphertext `ct` under `dk`: J(z || ct), z being dk's last 32 bytes. */ +std::array RejectionSecret(const DecapsulationKey& dk, std::span ct) +{ + std::vector input(dk.Bytes().end() - 32, dk.Bytes().end()); + input.insert(input.end(), ct.begin(), ct.end()); + return Shake256To32(input); +} + +/** Overwrite coefficient `index` of polynomial `poly` in an encapsulation key (FIPS 203 ByteEncode_12). */ +void SetCoefficient(PublicKey& ek, int poly, int index, uint16_t value) +{ + uint8_t* group{ek.data() + 384 * poly + 3 * (index / 2)}; + if (index % 2 == 0) { + group[0] = uint8_t(value); + group[1] = uint8_t((group[1] & 0xf0) | (value >> 8)); + } else { + group[1] = uint8_t((group[1] & 0x0f) | ((value & 0x0f) << 4)); + group[2] = uint8_t(value >> 4); + } +} + +/** + * The 1040 invalid ML-KEM-1024 encapsulation keys of C2SP CCTV + * ML-KEM/modulus/ML-KEM-1024.txt, in file order, rebuilt from the key they + * share, as CCTV's modulus.go generates them. + */ +std::vector CctvModulusKeys(const PublicKey& base) +{ + std::vector keys; + const auto add{[&](int poly, int index, uint16_t value) { + keys.push_back(base); + SetCoefficient(keys.back(), poly, index, value); + }}; + for (int i{0}; i < MLKEM_K; ++i) { + add(i, 0, MLKEM_Q); + add(i, 255, MLKEM_Q); + add(i, 0, (1 << 12) - 1); + add(i, 255, (1 << 12) - 1); + } + int i{0}, j{0}; + uint16_t x{MLKEM_Q}; + bool done_values{false}, done_positions{false}; + while (true) { + add(i, j, x); + if (++x == (1 << 12)) { + x = MLKEM_Q; + done_values = true; + } + if (++j == MLKEM_N) { + j = 0; + ++i; + } + if (i == MLKEM_K) { + i = 0; + done_positions = true; + } + if (done_values && done_positions) break; + } + return keys; +} + +} // namespace + +BOOST_FIXTURE_TEST_SUITE(mlkem_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(acvp_keygen_1024) +{ + const UniValue vectors{ReadVectors()}; + const UniValue& cases{vectors["acvp"]["keyGen"]}; + BOOST_REQUIRE(!cases.empty()); + for (const UniValue& tc : cases.getValues()) { + BOOST_TEST_CONTEXT("ACVP keyGen tcId " << tc["tcId"].getInt()) + { + std::array seed; + const auto d{FromHex<32>(tc["d"])}; + const auto z{FromHex<32>(tc["z"])}; + std::ranges::copy(d, seed.begin()); + std::ranges::copy(z, seed.begin() + 32); + const auto expected_ek{FromHex(tc["ek"])}; + const auto expected_dk{FromHex(tc["dk"])}; + ForEachBackend([&] { + PublicKey ek; + DecapsulationKey dk; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + BOOST_CHECK(ek == expected_ek); + BOOST_CHECK(std::ranges::equal(dk.Bytes(), expected_dk)); + }); + } + } +} + +BOOST_AUTO_TEST_CASE(acvp_encap_decap_1024) +{ + const UniValue vectors{ReadVectors()}; + const UniValue& acvp{vectors["acvp"]}; + + BOOST_REQUIRE(!acvp["encapsulation"].empty()); + for (const UniValue& tc : acvp["encapsulation"].getValues()) { + BOOST_TEST_CONTEXT("ACVP encapsulation tcId " << tc["tcId"].getInt()) + { + const auto ek{FromHex(tc["ek"])}; + const auto coins{FromHex(tc["m"])}; + const auto expected_ct{FromHex(tc["c"])}; + const auto expected_ss{FromHex(tc["k"])}; + ForEachBackend([&] { + Ciphertext ct; + SharedSecret ss; + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + BOOST_CHECK(ct == expected_ct); + BOOST_CHECK(std::ranges::equal(ss.Bytes(), expected_ss)); + }); + } + } + + // Both "valid decapsulation" and "modified ciphertext" cases decapsulate + // successfully; the latter yield the implicit-rejection secret. + BOOST_REQUIRE(!acvp["decapsulation"].empty()); + for (const UniValue& tc : acvp["decapsulation"].getValues()) { + BOOST_TEST_CONTEXT("ACVP decapsulation tcId " << tc["tcId"].getInt() << " (" << tc["reason"].get_str() << ")") + { + const DecapsulationKey dk{DecapsulationKeyFromHex(tc["dk"])}; + const auto ct{FromHex(tc["c"])}; + const auto expected_ss{FromHex(tc["k"])}; + ForEachBackend([&] { + SharedSecret ss; + BOOST_REQUIRE_EQUAL(Decaps(dk, ct, ss), Error::NONE); + BOOST_CHECK(std::ranges::equal(ss.Bytes(), expected_ss)); + }); + } + } + + BOOST_REQUIRE(!acvp["encapsulationKeyCheck"].empty()); + for (const UniValue& tc : acvp["encapsulationKeyCheck"].getValues()) { + BOOST_TEST_CONTEXT("ACVP encapsulationKeyCheck tcId " << tc["tcId"].getInt() << " (" << tc["reason"].get_str() << ")") + { + const auto ek{FromHex(tc["ek"])}; + const Error expected{tc["testPassed"].get_bool() ? Error::NONE : Error::INVALID_PUBLIC_KEY}; + ForEachBackend([&] { + BOOST_CHECK_EQUAL(CheckPublicKey(ek), expected); + const std::array coins{}; + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), expected); + }); + } + } + + BOOST_REQUIRE(!acvp["decapsulationKeyCheck"].empty()); + for (const UniValue& tc : acvp["decapsulationKeyCheck"].getValues()) { + BOOST_TEST_CONTEXT("ACVP decapsulationKeyCheck tcId " << tc["tcId"].getInt() << " (" << tc["reason"].get_str() << ")") + { + const DecapsulationKey dk{DecapsulationKeyFromHex(tc["dk"])}; + const Error expected{tc["testPassed"].get_bool() ? Error::NONE : Error::INVALID_SECRET_KEY}; + ForEachBackend([&] { + const Ciphertext ct{}; + SharedSecret ss; + BOOST_CHECK_EQUAL(Decaps(dk, ct, ss), expected); + }); + } + } +} + +BOOST_AUTO_TEST_CASE(public_key_modulus) +{ + const UniValue vectors{ReadVectors()}; + const UniValue& cctv{vectors["cctv_modulus"]}; + const PublicKey base{FromHex(cctv["base_ek"])}; + const std::array coins{}; + + // The rebuilt keys are exactly CCTV's file. + const std::vector invalid_keys{CctvModulusKeys(base)}; + BOOST_REQUIRE_EQUAL(invalid_keys.size(), size_t(cctv["count"].getInt())); + CSHA256 file_hash; + for (const PublicKey& ek : invalid_keys) { + const std::string line{HexStr(ek) + "\n"}; + file_hash.Write(reinterpret_cast(line.data()), line.size()); + } + std::array digest; + file_hash.Finalize(digest.data()); + BOOST_REQUIRE_EQUAL(HexStr(digest), cctv["sha256_of_txt"].get_str()); + + ForEachBackend([&] { + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(CheckPublicKey(base), Error::NONE); + BOOST_CHECK_EQUAL(Encaps(base, coins, ct, ss), Error::NONE); + + for (size_t n{0}; n < invalid_keys.size(); ++n) { + BOOST_TEST_CONTEXT("CCTV modulus key " << n) + { + BOOST_CHECK_EQUAL(CheckPublicKey(invalid_keys[n]), Error::INVALID_PUBLIC_KEY); + ct.fill(0xa5); + std::ranges::fill(ss.Bytes(), 0xa5); + BOOST_CHECK_EQUAL(Encaps(invalid_keys[n], coins, ct, ss), Error::INVALID_PUBLIC_KEY); + BOOST_CHECK(AllBytesAre(ct, 0)); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + } + + // The boundary: q - 1 = 3328 is the largest valid coefficient, q = 3329 the smallest invalid one. + for (int poly{0}; poly < MLKEM_K; ++poly) { + for (const int index : {0, 1, 128, 255}) { + BOOST_TEST_CONTEXT("coefficient " << index << " of polynomial " << poly) + { + PublicKey ek{base}; + SetCoefficient(ek, poly, index, MLKEM_Q - 1); + BOOST_CHECK_EQUAL(CheckPublicKey(ek), Error::NONE); + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + SetCoefficient(ek, poly, index, MLKEM_Q); + BOOST_CHECK_EQUAL(CheckPublicKey(ek), Error::INVALID_PUBLIC_KEY); + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::INVALID_PUBLIC_KEY); + } + } + } + }); +} + +BOOST_AUTO_TEST_CASE(implicit_rejection) +{ + // The reference: ACVP's modified-ciphertext cases equal J(z || c). + const UniValue vectors{ReadVectors()}; + int modified_cases{0}; + for (const UniValue& tc : vectors["acvp"]["decapsulation"].getValues()) { + if (tc["reason"].get_str() != "modified ciphertext") continue; + const DecapsulationKey dk{DecapsulationKeyFromHex(tc["dk"])}; + const auto ct{FromHex(tc["c"])}; + BOOST_CHECK(RejectionSecret(dk, ct) == FromHex(tc["k"])); + ++modified_cases; + } + BOOST_REQUIRE_GT(modified_cases, 0); + + std::array seed; + for (size_t i{0}; i < seed.size(); ++i) seed[i] = uint8_t(i); + std::array coins; + coins.fill(0x42); + + ForEachBackend([&] { + PublicKey ek; + DecapsulationKey dk; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + Ciphertext ct; + SharedSecret sent; + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, sent), Error::NONE); + SharedSecret received; + BOOST_REQUIRE_EQUAL(Decaps(dk, ct, received), Error::NONE); + BOOST_CHECK(std::ranges::equal(sent.Bytes(), received.Bytes())); + + for (const auto& [byte, bit] : std::vector>{{0, 0}, {784, 3}, {1407, 7}, {CIPHERTEXT_BYTES - 1, 7}}) { + BOOST_TEST_CONTEXT("flipped bit " << bit << " of ciphertext byte " << byte) + { + Ciphertext corrupted{ct}; + corrupted[byte] ^= uint8_t(1 << bit); + SharedSecret rejected; + BOOST_REQUIRE_EQUAL(Decaps(dk, corrupted, rejected), Error::NONE); + BOOST_CHECK(!std::ranges::equal(rejected.Bytes(), sent.Bytes())); + BOOST_CHECK(std::ranges::equal(rejected.Bytes(), RejectionSecret(dk, corrupted))); + } + } + }); +} + +BOOST_AUTO_TEST_CASE(wrapper_lifetimes_and_errors) +{ + const UniValue vectors{ReadVectors()}; + std::array seed; + seed.fill(0x07); + const std::array coins{}; + PublicKey ek; + DecapsulationKey dk; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + + // An invalid decapsulation key (failed hash check) is its own error, and clears the secret. + for (const UniValue& tc : vectors["acvp"]["decapsulationKeyCheck"].getValues()) { + if (tc["testPassed"].get_bool()) continue; + const DecapsulationKey bad_dk{DecapsulationKeyFromHex(tc["dk"])}; + SharedSecret ss; + std::ranges::fill(ss.Bytes(), 0x5a); + BOOST_CHECK_EQUAL(Decaps(bad_dk, Ciphertext{}, ss), Error::INVALID_SECRET_KEY); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + + // Injected library results: anything but the documented invalid-key codes + // is INTERNAL, and every failure clears the outputs the library wrote. + { + InjectResultForTesting inject{Operation::KEYGEN, upstream::ERR_FAIL}; + PublicKey ek2; + DecapsulationKey dk2; + BOOST_CHECK_EQUAL(KeyGen(seed, ek2, dk2), Error::INTERNAL); + BOOST_CHECK(AllBytesAre(ek2, 0)); + BOOST_CHECK(AllBytesAre(dk2.Bytes(), 0)); + } + { + InjectResultForTesting inject{Operation::KEYGEN, upstream::ERR_INVALID_PK}; + PublicKey ek2; + DecapsulationKey dk2; + BOOST_CHECK_EQUAL(KeyGen(seed, ek2, dk2), Error::INTERNAL); + BOOST_CHECK(AllBytesAre(dk2.Bytes(), 0)); + } + for (const auto& [result, expected] : std::vector>{ + {upstream::ERR_INVALID_PK, Error::INVALID_PUBLIC_KEY}, + {upstream::ERR_INVALID_SK, Error::INTERNAL}, + {upstream::ERR_OUT_OF_MEMORY, Error::INTERNAL}}) { + BOOST_TEST_CONTEXT("upstream result " << result) + { + { + InjectResultForTesting inject{Operation::CHECK_PUBLIC_KEY, result}; + BOOST_CHECK_EQUAL(CheckPublicKey(ek), expected); + } + InjectResultForTesting inject{Operation::ENCAPS, result}; + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), expected); + BOOST_CHECK(AllBytesAre(ct, 0)); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + } + for (const auto& [result, expected] : std::vector>{ + {upstream::ERR_INVALID_SK, Error::INVALID_SECRET_KEY}, + {upstream::ERR_INVALID_PK, Error::INTERNAL}, + {upstream::ERR_RNG_FAIL, Error::INTERNAL}, + {upstream::ERR_PCT_FAIL, Error::INTERNAL}}) { + BOOST_TEST_CONTEXT("upstream result " << result) + { + InjectResultForTesting inject{Operation::DECAPS, result}; + SharedSecret ss; + BOOST_CHECK_EQUAL(Decaps(dk, Ciphertext{}, ss), expected); + BOOST_CHECK(AllBytesAre(ss.Bytes(), 0)); + } + } + { + // Guards nest and restore what they replaced. + InjectResultForTesting outer{Operation::ENCAPS, upstream::ERR_FAIL}; + { + InjectResultForTesting inner{Operation::ENCAPS, upstream::ERR_INVALID_PK}; + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::INVALID_PUBLIC_KEY); + } + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(Encaps(ek, coins, ct, ss), Error::INTERNAL); + } + // Without injection every operation succeeds again. + Ciphertext ct; + SharedSecret ss; + BOOST_CHECK_EQUAL(CheckPublicKey(ek), Error::NONE); + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + + // Moves transfer the secret and clear the source; Clear() wipes. + DecapsulationKey moved_dk{std::move(dk)}; + BOOST_CHECK(AllBytesAre(dk.Bytes(), 0)); // NOLINT(bugprone-use-after-move) + SharedSecret decapsulated; + BOOST_REQUIRE_EQUAL(Decaps(moved_dk, ct, decapsulated), Error::NONE); + BOOST_CHECK(std::ranges::equal(decapsulated.Bytes(), ss.Bytes())); + + SharedSecret assigned; + assigned = std::move(decapsulated); + BOOST_CHECK(std::ranges::equal(assigned.Bytes(), ss.Bytes())); + BOOST_CHECK(AllBytesAre(decapsulated.Bytes(), 0)); // NOLINT(bugprone-use-after-move) + SharedSecret& alias{assigned}; + assigned = std::move(alias); + BOOST_CHECK(std::ranges::equal(assigned.Bytes(), ss.Bytes())); + assigned.Clear(); + BOOST_CHECK(AllBytesAre(assigned.Bytes(), 0)); + + // Destruction wipes the bytes. + alignas(SharedSecret) unsigned char storage[sizeof(SharedSecret)]; + SharedSecret* secret{new (storage) SharedSecret}; + std::ranges::fill(secret->Bytes(), 0xc3); + secret->~SharedSecret(); +#ifdef MLKEM_TESTS_MSAN + // MemorySanitizer marks a destroyed object's storage uninitialized; read + // the bytes the destructor left behind anyway. + __msan_unpoison(storage, sizeof(storage)); +#endif + BOOST_CHECK(AllBytesAre(storage, 0)); +} + +BOOST_AUTO_TEST_CASE(backend_selection) +{ + const std::string_view compiled_arith{qbit_mlkem_compiled_arith_backend()}; + const std::string_view compiled_keccak{qbit_mlkem_compiled_keccak_backend()}; + const BackendNames active{GetBackendNames()}; + BOOST_TEST_MESSAGE("compiled " << compiled_arith << "/" << compiled_keccak << ", active " << active.arith << "/" << active.keccak); + + // The configure step's decision reaches the compiled library. +#if ENABLE_MLKEM_NATIVE && (defined(__x86_64__) || defined(_M_X64)) + BOOST_CHECK_MESSAGE(compiled_arith == "x86_64-avx2" && compiled_keccak == "x86_64-avx2", + "FAIL: configure chose native x86_64, but the library was compiled with " << compiled_arith << "/" << compiled_keccak + << ". Cause: cmake/mlkem-native.cmake and src/crypto/mlkem_config.h disagree. Fix: compare the probes in mlkem_native_detect_arch with the header's conditions."); +#ifdef HAVE_BUILTIN_CPU_SUPPORTS + // On a host with every extension the x86_64 assembly executes, native code + // must be what runs; without any one of them (BMI2 or POPCNT masked in a + // virtual machine, say), portable C must. + __builtin_cpu_init(); + const bool x86_native{__builtin_cpu_supports("avx2") && __builtin_cpu_supports("ssse3") && __builtin_cpu_supports("sse4.1") && + __builtin_cpu_supports("popcnt") && __builtin_cpu_supports("bmi2")}; + const std::string_view expected{x86_native ? "x86_64-avx2" : "portable"}; + BOOST_CHECK_MESSAGE(active.arith == expected && active.keccak == expected, + "FAIL: active backend " << active.arith << "/" << active.keccak << ", expected " << expected + << ". Cause: cached detection (cpu_features::HasMlkemX86Native) or the capability hook disagrees with the CPU. Fix: check cpu_features_tests and qbit_mlkem_has_avx2."); +#endif +#elif defined(__aarch64__) && defined(__APPLE__) && ENABLE_MLKEM_NATIVE + BOOST_CHECK_EQUAL(compiled_arith, "aarch64-neon"); + BOOST_CHECK_EQUAL(compiled_keccak, "aarch64"); + BOOST_CHECK_EQUAL(active.arith, "aarch64-neon"); + BOOST_CHECK_EQUAL(active.keccak, "aarch64"); +#else + // WITH_MLKEM_NATIVE=OFF, MemorySanitizer, or a target without a native + // backend, AArch64 ELF included: its assembly lacks BTI landing pads. + BOOST_CHECK_MESSAGE(!ENABLE_MLKEM_NATIVE, "native build for a target without a qbit backend"); + BOOST_CHECK_EQUAL(compiled_arith, "portable"); + BOOST_CHECK_EQUAL(compiled_keccak, "portable"); + BOOST_CHECK_EQUAL(active.arith, "portable"); + BOOST_CHECK_EQUAL(active.keccak, "portable"); +#endif + + // The portable guard forces portable C, nests, and restores what it replaced. + { + ForcePortableForTesting outer; + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, "portable"); + { + ForcePortableForTesting inner; + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + } + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, "portable"); + } + BOOST_CHECK_EQUAL(GetBackendNames().arith, active.arith); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, active.keccak); + + // The startup override. + InitializeRuntime(/*force_portable=*/true); + BOOST_CHECK_EQUAL(GetBackendNames().arith, "portable"); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, "portable"); + InitializeRuntime(/*force_portable=*/false); + BOOST_CHECK_EQUAL(GetBackendNames().arith, active.arith); + BOOST_CHECK_EQUAL(GetBackendNames().keccak, active.keccak); +} + +BOOST_AUTO_TEST_CASE(override_changed_by_another_thread) +{ + const BackendNames before{GetBackendNames()}; + // The outputs with nothing changing the override; every backend gives them. + std::array seed; + for (size_t i{0}; i < seed.size(); ++i) seed[i] = uint8_t(i * 7 + 1); + std::array coins; + for (size_t i{0}; i < coins.size(); ++i) coins[i] = uint8_t(i * 3 + 2); + PublicKey expected_ek; + DecapsulationKey expected_dk; + Ciphertext expected_ct; + SharedSecret expected_ss; + BOOST_REQUIRE_EQUAL(KeyGen(seed, expected_ek, expected_dk), Error::NONE); + BOOST_REQUIRE_EQUAL(Encaps(expected_ek, coins, expected_ct, expected_ss), Error::NONE); + + // Another thread flips the override as fast as it can while this one runs + // operations. Each operation must finish on the backend it started with: + // on x86_64, one that mixed backends would give wrong outputs. + std::atomic_bool stop{false}; + std::atomic flips{0}; + std::thread flipper{[&] { + while (!stop.load(std::memory_order_relaxed)) { + ForcePortableForTesting portable; + flips.fetch_add(1, std::memory_order_relaxed); + } + }}; + while (flips.load(std::memory_order_relaxed) == 0) std::this_thread::yield(); + int wrong{0}; + int portable_seen{0}; + constexpr int ROUNDS{50}; + for (int round{0}; round < ROUNDS; ++round) { + PublicKey ek; + DecapsulationKey dk; + Ciphertext ct; + SharedSecret ss; + SharedSecret decapsulated; + const bool ok{KeyGen(seed, ek, dk) == Error::NONE && Encaps(ek, coins, ct, ss) == Error::NONE && + Decaps(dk, ct, decapsulated) == Error::NONE}; + if (!ok || ek != expected_ek || !std::ranges::equal(dk.Bytes(), expected_dk.Bytes()) || ct != expected_ct || + !std::ranges::equal(ss.Bytes(), expected_ss.Bytes()) || !std::ranges::equal(decapsulated.Bytes(), expected_ss.Bytes())) { + ++wrong; + } + if (GetBackendNames().arith == "portable") ++portable_seen; + } + stop.store(true, std::memory_order_relaxed); + flipper.join(); + BOOST_TEST_MESSAGE(flips.load() << " flips; portable active at " << portable_seen << " of " << ROUNDS << " checks"); + BOOST_CHECK_MESSAGE(wrong == 0, "FAIL: " << wrong << " of " << ROUNDS << " rounds gave wrong outputs while another thread changed the override. " + << "Cause: an operation mixed backends. Fix: the capability hooks in src/crypto/mlkem.cpp must read the copy taken at entry."); + BOOST_CHECK_EQUAL(GetBackendNames().arith, before.arith); +} + +BOOST_AUTO_TEST_SUITE_END() From 85af85d990335aacdff0f987aa00a45b969dd941 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 13:26:10 -0400 Subject: [PATCH 05/20] fuzz: exercise ML-KEM validation and backend equivalence Add two fuzz targets over the mlkem:: wrapper: - mlkem: honest key generation, encapsulation and decapsulation agree; an arbitrary encapsulation key is accepted exactly when an independent modulus check passes, and a rejected one leaves no output; any ciphertext of the right length decapsulates to a secret that differs from the honest one (implicit rejection is never proof of validity); an arbitrary decapsulation key is rejected exactly when an independent SHA3-256 hash check fails, and the secret is then cleared. - mlkem_backend_diff: the detected backend and forced portable C give equal statuses and outputs for the same inputs, and the override is restored afterwards. Both honor MLKEM_FORCE_PORTABLE=1. They join the required qbit corpus with nine generated seeds each (CCTV modulus key and 3328/3329/4095 coefficients, NIST ACVP decapsulation vectors, a ciphertext bit flip). CI replays them a second time with MLKEM_FORCE_PORTABLE=1 and includes them in the mutation phase; the PQ CI contract test checks that wiring. UBSan's -fsanitize=integer checks are suppressed for the vendored tree, as for the other external libraries. Part of #184 (part 3, #190, D5). Co-Authored-By: Claude Opus 5.5 --- ci/checks/test_ci_pq_contract.py | 68 ++++++ ci/test/03_test_script.sh | 12 +- doc/development/fuzzing.md | 11 +- src/test/fuzz/CMakeLists.txt | 1 + src/test/fuzz/mlkem.cpp | 228 ++++++++++++++++++ test/fuzz/qbit_corpora/MANIFEST.json | 130 ++++++++++ test/fuzz/qbit_corpora/README.md | 14 +- test/fuzz/qbit_corpora/generate_seeds.py | 95 ++++++++ .../mlkem/acvp-modified-ciphertext | Bin 0 -> 6400 bytes .../qbit_corpora/mlkem/acvp-modified-key-hash | Bin 0 -> 6400 bytes .../mlkem/acvp-valid-decapsulation | Bin 0 -> 6400 bytes test/fuzz/qbit_corpora/mlkem/cctv-base-key | Bin 0 -> 1664 bytes test/fuzz/qbit_corpora/mlkem/coefficient-3328 | Bin 0 -> 1664 bytes test/fuzz/qbit_corpora/mlkem/coefficient-3329 | Bin 0 -> 1664 bytes .../mlkem/honest-ciphertext-bit-flip | Bin 0 -> 6403 bytes .../fuzz/qbit_corpora/mlkem/honest-round-trip | Bin 0 -> 96 bytes .../qbit_corpora/mlkem/last-coefficient-4095 | Bin 0 -> 1664 bytes .../acvp-modified-ciphertext | Bin 0 -> 6400 bytes .../mlkem_backend_diff/acvp-modified-key-hash | Bin 0 -> 6400 bytes .../acvp-valid-decapsulation | Bin 0 -> 6400 bytes .../mlkem_backend_diff/cctv-base-key | Bin 0 -> 1664 bytes .../mlkem_backend_diff/coefficient-3328 | Bin 0 -> 1664 bytes .../mlkem_backend_diff/coefficient-3329 | Bin 0 -> 1664 bytes .../honest-ciphertext-bit-flip | Bin 0 -> 6403 bytes .../mlkem_backend_diff/honest-round-trip | Bin 0 -> 96 bytes .../mlkem_backend_diff/last-coefficient-4095 | Bin 0 -> 1664 bytes test/fuzz/qbit_corpora/overlay.py | 2 + test/fuzz/test_qbit_fuzz_runner.py | 2 +- test/fuzz/test_runner.py | 2 + test/sanitizer_suppressions/ubsan | 5 + 30 files changed, 564 insertions(+), 6 deletions(-) create mode 100644 src/test/fuzz/mlkem.cpp create mode 100644 test/fuzz/qbit_corpora/mlkem/acvp-modified-ciphertext create mode 100644 test/fuzz/qbit_corpora/mlkem/acvp-modified-key-hash create mode 100644 test/fuzz/qbit_corpora/mlkem/acvp-valid-decapsulation create mode 100644 test/fuzz/qbit_corpora/mlkem/cctv-base-key create mode 100644 test/fuzz/qbit_corpora/mlkem/coefficient-3328 create mode 100644 test/fuzz/qbit_corpora/mlkem/coefficient-3329 create mode 100644 test/fuzz/qbit_corpora/mlkem/honest-ciphertext-bit-flip create mode 100644 test/fuzz/qbit_corpora/mlkem/honest-round-trip create mode 100644 test/fuzz/qbit_corpora/mlkem/last-coefficient-4095 create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-ciphertext create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-key-hash create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-valid-decapsulation create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/cctv-base-key create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3328 create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3329 create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/honest-ciphertext-bit-flip create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/honest-round-trip create mode 100644 test/fuzz/qbit_corpora/mlkem_backend_diff/last-coefficient-4095 diff --git a/ci/checks/test_ci_pq_contract.py b/ci/checks/test_ci_pq_contract.py index c54bbe596c..3228aac485 100755 --- a/ci/checks/test_ci_pq_contract.py +++ b/ci/checks/test_ci_pq_contract.py @@ -1002,5 +1002,73 @@ def main() -> int: self.assertIn("unreadable ctest test listing", summary.read_text(encoding="utf8")) + +class FuzzReplayContractTest(unittest.TestCase): + """The fuzz phase of 03_test_script.sh replays the ML-KEM targets on portable C too (#190 D5).""" + + STUB_RUNNER = r'''#!/usr/bin/env python3 +import json, os, sys +with open(os.environ["STUB_LOG"], "a", encoding="utf8") as log: + log.write(json.dumps({"args": sys.argv[1:], "MLKEM_FORCE_PORTABLE": os.environ.get("MLKEM_FORCE_PORTABLE")}) + "\n") +''' + + @staticmethod + def fuzz_block() -> str: + """The last RUN_FUZZ_TESTS block of 03_test_script.sh: the replay and mutation phase.""" + lines = TEST_SCRIPT.read_text(encoding="utf8").splitlines() + first = max(index for index, line in enumerate(lines) if line == 'if [ "$RUN_FUZZ_TESTS" = "true" ]; then') + last = next(index for index in range(first + 1, len(lines)) if lines[index] == "fi") + return "\n".join(lines[first:last + 1]) + "\n" + + @staticmethod + def required_targets() -> list[str]: + """QBIT_REQUIRED_CORPUS_TARGETS from test/fuzz/test_runner.py, read without importing it.""" + import ast + tree = ast.parse((REPO_ROOT / "test" / "fuzz" / "test_runner.py").read_text(encoding="utf8")) + for node in tree.body: + if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "QBIT_REQUIRED_CORPUS_TARGETS" for t in node.targets): + return list(ast.literal_eval(node.value)) + raise AssertionError("QBIT_REQUIRED_CORPUS_TARGETS not found") + + def run_block(self, mutate_min_time: str) -> list[dict[str, Any]]: + with tempfile.TemporaryDirectory(prefix="ci-fuzz-") as tmp: + root = Path(tmp) + runner = root / "build" / "test" / "fuzz" / "test_runner.py" + runner.parent.mkdir(parents=True) + runner.write_text(self.STUB_RUNNER, encoding="utf8") + runner.chmod(0o755) + log = root / "calls.jsonl" + log.touch() + env = {"PATH": os.environ["PATH"], "HOME": tmp, "LC_ALL": "C", "RUN_FUZZ_TESTS": "true", + "BASE_BUILD_DIR": str(root / "build"), "DEPENDS_DIR": str(root / "depends"), "HOST": "x86_64-pc-linux-gnu", + "MAKEJOBS": "-j2", "DIR_FUZZ_IN": str(root / "corpora"), "FUZZ_TESTS_CONFIG": "", + "QBIT_FUZZ_MUTATE_MIN_TIME": mutate_min_time, "STUB_LOG": str(log)} + script = root / "block.sh" + script.write_text("set -ex\n" + self.fuzz_block(), encoding="utf8") + completed = subprocess.run(["bash", str(script)], env=env, text=True, capture_output=True) + self.assertEqual(completed.returncode, 0, completed.stderr) + return [json.loads(line) for line in log.read_text(encoding="utf8").splitlines()] + + def test_mlkem_targets_replay_on_portable_c(self) -> None: + replay, portable = self.run_block(mutate_min_time="") + self.assertIn("--require_qbit_corpus", replay["args"]) + self.assertIsNone(replay["MLKEM_FORCE_PORTABLE"]) + self.assertEqual(portable["MLKEM_FORCE_PORTABLE"], "1") + self.assertEqual(portable["args"][-2:], ["mlkem", "mlkem_backend_diff"]) + corpus = [arg for arg in replay["args"] if arg.endswith("/corpora")] + self.assertEqual(len(corpus), 1, replay["args"]) + self.assertIn(corpus[0], portable["args"], "the portable replay uses the same corpus") + + def test_mutation_phase_covers_every_required_target(self) -> None: + calls = self.run_block(mutate_min_time="60") + self.assertEqual(len(calls), 3) + mutate = calls[2] + self.assertIn("--mutate_min_time=60", mutate["args"]) + self.assertIsNone(mutate["MLKEM_FORCE_PORTABLE"]) + targets = self.required_targets() + self.assertIn("mlkem", targets) + self.assertEqual(mutate["args"][-len(targets):], targets) + + if __name__ == "__main__": unittest.main() diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 20dad10606..6399230d18 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -350,6 +350,16 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then "${DIR_FUZZ_IN}" \ --empty_min_time=60 \ --require_qbit_corpus + # Replay the ML-KEM targets again on portable C, which they select at start-up. + # shellcheck disable=SC2086 + MLKEM_FORCE_PORTABLE=1 \ + LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \ + "${BASE_BUILD_DIR}/test/fuzz/test_runner.py" \ + ${FUZZ_TESTS_CONFIG} \ + "${MAKEJOBS}" \ + -l DEBUG \ + "${DIR_FUZZ_IN}" \ + mlkem mlkem_backend_diff if [ -n "${QBIT_FUZZ_MUTATE_MIN_TIME}" ]; then # Seeded mutation phase over the qbit seed corpora, after the replay above. # shellcheck disable=SC2086 @@ -361,6 +371,6 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then --require_qbit_corpus \ --mutate_min_time="${QBIT_FUZZ_MUTATE_MIN_TIME}" \ "${DIR_FUZZ_IN}" \ - asert_chain_transition asert_edge_cases asert_math auxpow p2mr_script pqc + asert_chain_transition asert_edge_cases asert_math auxpow mlkem mlkem_backend_diff p2mr_script pqc fi fi diff --git a/doc/development/fuzzing.md b/doc/development/fuzzing.md index e86895f21e..477bbc70ba 100644 --- a/doc/development/fuzzing.md +++ b/doc/development/fuzzing.md @@ -41,7 +41,8 @@ $ ctest --test-dir build_fuzz -R '^required_fuzz_targets$' --output-on-failure ## qbit seed corpora `qa-assets` has no inputs for `asert_chain_transition`, `asert_edge_cases`, -`asert_math`, `auxpow`, `p2mr_script` or `pqc`. Generated seeds for these +`asert_math`, `auxpow`, `mlkem`, `mlkem_backend_diff`, `p2mr_script` or `pqc`. +Generated seeds for these targets, their manifest and the input format are described in [`test/fuzz/qbit_corpora/README.md`](../../test/fuzz/qbit_corpora/README.md). CI copies them into the `qa-assets` corpus and requires that they are replayed: @@ -51,17 +52,21 @@ $ test/fuzz/qbit_corpora/overlay.py qa-assets/fuzz_corpora $ build_fuzz/test/fuzz/test_runner.py --require_qbit_corpus qa-assets/fuzz_corpora ``` -`--require_qbit_corpus` fails if any of the six targets is not compiled or not +`--require_qbit_corpus` fails if any of the eight targets is not compiled or not selected (including through `--exclude`), has no regular input files, or is not reported as replayed by the fuzz executable. The runner prints the number of input files present and replayed for each target. +The ML-KEM targets use native code where the CPU has it. CI replays `mlkem` and +`mlkem_backend_diff` a second time with `MLKEM_FORCE_PORTABLE=1`, which makes +them run portable C throughout. + With a libFuzzer build, `--mutate_min_time=` replaces the single replay with a mutation phase of that many seconds per target. The phase starts from the corpus, writes new inputs to a temporary directory and leaves the corpus unchanged. It fails on non-libFuzzer builds. The nightly native fuzz job sets `QBIT_FUZZ_MUTATE_MIN_TIME`, which makes `ci/test/03_test_script.sh` run -this phase for the six targets after the normal replay. +this phase for the eight targets after the normal replay. The budget is libFuzzer's `-max_total_time`, which counts time spent loading the seed corpus (libFuzzer's `INITED` line) as well as mutation. A run can end diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index eb1c06226e..bbaa22cdfe 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -66,6 +66,7 @@ add_executable(fuzz message.cpp miniscript.cpp minisketch.cpp + mlkem.cpp mini_miner.cpp muhash.cpp multiplication_overflow.cpp diff --git a/src/test/fuzz/mlkem.cpp b/src/test/fuzz/mlkem.cpp new file mode 100644 index 0000000000..417b0d1ab6 --- /dev/null +++ b/src/test/fuzz/mlkem.cpp @@ -0,0 +1,228 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mlkem; + +namespace { + +void Initialize() +{ + // MLKEM_FORCE_PORTABLE=1 runs every operation on portable C. + const char* force_portable{std::getenv("MLKEM_FORCE_PORTABLE")}; + InitializeRuntime(force_portable != nullptr && std::string_view{force_portable} == "1"); +} + +template +std::array ConsumeArray(FuzzedDataProvider& provider) +{ + std::array out{}; + const std::vector bytes{provider.ConsumeBytes(N)}; + std::ranges::copy(bytes, out.begin()); + return out; +} + +/** + * Inputs of both targets, in this order (bytes missing at the end of the input + * are zero): the key generation seed d || z, the encapsulation coins, an + * arbitrary encapsulation key, an arbitrary ciphertext and an arbitrary + * decapsulation key; then, from the back, whether to flip one bit of the + * honest ciphertext, and which. + */ +struct Inputs { + std::array seed; + std::array coins; + PublicKey ek; + Ciphertext ct; + DecapsulationKey dk; + bool flip; + size_t flip_bit; +}; + +Inputs ConsumeInputs(FuzzedDataProvider& provider) +{ + Inputs in{ + .seed = ConsumeArray(provider), + .coins = ConsumeArray(provider), + .ek = ConsumeArray(provider), + .ct = ConsumeArray(provider), + .dk = {}, + .flip = false, + .flip_bit = 0, + }; + std::ranges::copy(ConsumeArray(provider), in.dk.Bytes().begin()); + in.flip = provider.ConsumeBool(); + in.flip_bit = provider.ConsumeIntegralInRange(0, CIPHERTEXT_BYTES * 8 - 1); + return in; +} + +/** FIPS 203 section 7.2 modulus check, written independently of the library. */ +bool PublicKeyInRange(const PublicKey& ek) +{ + for (size_t i{0}; i < 384 * 4; i += 3) { + const unsigned low{ek[i] | ((ek[i + 1] & 0x0fU) << 8)}; + const unsigned high{(ek[i + 1] >> 4) | (unsigned{ek[i + 2]} << 4)}; + if (low >= 3329 || high >= 3329) return false; + } + return true; +} + +/** FIPS 203 section 7.3 hash check, H(ek) stored in dk, using qbit's own SHA3-256. */ +bool SecretKeyHashMatches(const DecapsulationKey& dk) +{ + const std::span bytes{dk.Bytes()}; + std::array hash; + SHA3_256().Write(bytes.subspan(384 * 4, PUBLIC_KEY_BYTES)).Finalize(hash); + return std::ranges::equal(hash, bytes.subspan(384 * 4 + PUBLIC_KEY_BYTES, hash.size())); +} + +bool AllZero(std::span bytes) +{ + return std::ranges::all_of(bytes, [](uint8_t b) { return b == 0; }); +} + +void FlipBit(Ciphertext& ct, size_t bit) +{ + ct[bit / 8] ^= uint8_t(1U << (bit % 8)); +} + +/** Every observable result of one pass over the inputs. */ +std::vector RunAll(const Inputs& in) +{ + std::vector out; + const auto add_error{[&](Error error) { out.push_back(uint8_t(error)); }}; + const auto add_bytes{[&](std::span bytes) { out.insert(out.end(), bytes.begin(), bytes.end()); }}; + + PublicKey ek; + DecapsulationKey dk; + add_error(KeyGen(in.seed, ek, dk)); + add_bytes(ek); + add_bytes(dk.Bytes()); + Ciphertext ct; + SharedSecret ss; + add_error(Encaps(ek, in.coins, ct, ss)); + add_bytes(ct); + add_bytes(ss.Bytes()); + { + SharedSecret received; + add_error(Decaps(dk, ct, received)); + add_bytes(received.Bytes()); + } + if (in.flip) { + Ciphertext corrupted{ct}; + FlipBit(corrupted, in.flip_bit); + SharedSecret received; + add_error(Decaps(dk, corrupted, received)); + add_bytes(received.Bytes()); + } + + add_error(CheckPublicKey(in.ek)); + { + Ciphertext ct2; + SharedSecret ss2; + add_error(Encaps(in.ek, in.coins, ct2, ss2)); + add_bytes(ct2); + add_bytes(ss2.Bytes()); + } + { + SharedSecret received; + add_error(Decaps(dk, in.ct, received)); + add_bytes(received.Bytes()); + } + { + SharedSecret received; + add_error(Decaps(in.dk, in.ct, received)); + add_bytes(received.Bytes()); + } + return out; +} + +} // namespace + +FUZZ_TARGET(mlkem, .init = Initialize) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + const Inputs in{ConsumeInputs(provider)}; + + // Honest round trips agree. + PublicKey ek; + DecapsulationKey dk; + assert(KeyGen(in.seed, ek, dk) == Error::NONE); + assert(CheckPublicKey(ek) == Error::NONE); + Ciphertext ct; + SharedSecret sent; + assert(Encaps(ek, in.coins, ct, sent) == Error::NONE); + SharedSecret received; + assert(Decaps(dk, ct, received) == Error::NONE); + assert(std::ranges::equal(received.Bytes(), sent.Bytes())); + + // An arbitrary encapsulation key is accepted exactly when it passes the + // modulus check, and a rejected one leaves no output behind. + const Error check{CheckPublicKey(in.ek)}; + assert(check == (PublicKeyInRange(in.ek) ? Error::NONE : Error::INVALID_PUBLIC_KEY)); + { + Ciphertext ct2; + ct2.fill(0xa5); + SharedSecret ss2; + std::ranges::fill(ss2.Bytes(), 0xa5); + assert(Encaps(in.ek, in.coins, ct2, ss2) == check); + if (check != Error::NONE) assert(AllZero(ct2) && AllZero(ss2.Bytes())); + } + + // Decapsulating a ciphertext of the right length always yields a secret + // (implicit rejection). That is never proof the ciphertext was valid: any + // other ciphertext yields a different secret. + { + SharedSecret other; + assert(Decaps(dk, in.ct, other) == Error::NONE); + if (in.ct != ct) assert(!std::ranges::equal(other.Bytes(), sent.Bytes())); + } + if (in.flip) { + Ciphertext corrupted{ct}; + FlipBit(corrupted, in.flip_bit); + SharedSecret rejected; + assert(Decaps(dk, corrupted, rejected) == Error::NONE); + assert(!std::ranges::equal(rejected.Bytes(), sent.Bytes())); + } + + // An arbitrary decapsulation key is rejected exactly when its hash check + // fails, and a rejected one leaves no secret behind. + { + SharedSecret ss; + std::ranges::fill(ss.Bytes(), 0xa5); + const Error error{Decaps(in.dk, in.ct, ss)}; + assert(error == (SecretKeyHashMatches(in.dk) ? Error::NONE : Error::INVALID_SECRET_KEY)); + if (error != Error::NONE) assert(AllZero(ss.Bytes())); + } +} + +FUZZ_TARGET(mlkem_backend_diff, .init = Initialize) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + const Inputs in{ConsumeInputs(provider)}; + + // The detected backend and portable C agree on every status and output. + const BackendNames detected{GetBackendNames()}; + const std::vector native_results{RunAll(in)}; + { + ForcePortableForTesting portable; + assert(GetBackendNames().arith == "portable" && GetBackendNames().keccak == "portable"); + assert(RunAll(in) == native_results); + } + // The override is restored. + assert(GetBackendNames().arith == detected.arith && GetBackendNames().keccak == detected.keccak); +} diff --git a/test/fuzz/qbit_corpora/MANIFEST.json b/test/fuzz/qbit_corpora/MANIFEST.json index 887c3f231f..5265d09ab8 100644 --- a/test/fuzz/qbit_corpora/MANIFEST.json +++ b/test/fuzz/qbit_corpora/MANIFEST.json @@ -170,6 +170,136 @@ "semantics": "Payload validation, header validation and CheckBlock in one input." } ], + "mlkem": [ + { + "file": "honest-round-trip", + "format": "fdp", + "size": 96, + "sha256": "60563d6bf3a00873f259f212eb2916e929f9a2912b6738402345c16a2fc6cb51", + "semantics": "Key generation, encapsulation and decapsulation from fixed seed and coins agree; the all-zero encapsulation key is in range, and the all-zero decapsulation key fails its hash check." + }, + { + "file": "cctv-base-key", + "format": "fdp", + "size": 1664, + "sha256": "fc04cec04516e98b8c4c8c855bc74c30e5d672a04d7be3a44415274c05c1ff4c", + "semantics": "The key shared by the C2SP CCTV modulus vectors, every coefficient in range: accepted." + }, + { + "file": "coefficient-3328", + "format": "fdp", + "size": 1664, + "sha256": "6c274898e6747f79cb4fec2dde374e46f37e578d1e4ee605e0a8ce9c377212b0", + "semantics": "First coefficient q - 1 = 3328, the largest valid value: accepted." + }, + { + "file": "coefficient-3329", + "format": "fdp", + "size": 1664, + "sha256": "9dea64bc7eef0413f8c5bdfea2f74b4cf1acb148ed1bfa846935db0de425eb2e", + "semantics": "First coefficient q = 3329, the smallest invalid value: rejected, outputs cleared." + }, + { + "file": "last-coefficient-4095", + "format": "fdp", + "size": 1664, + "sha256": "465c31c294e5e8a5e0effcedf2e346b2de1643635a66309d7f9856c3fb9a0a42", + "semantics": "Last coefficient of the last polynomial 4095: rejected, outputs cleared." + }, + { + "file": "acvp-valid-decapsulation", + "format": "fdp", + "size": 6400, + "sha256": "22599a5c645de5def70e8e074118f35683d5a324fbc17cd817cc5623c19d5746", + "semantics": "NIST ACVP decapsulation tcId 97 (valid): its key, ciphertext and decapsulation key." + }, + { + "file": "acvp-modified-ciphertext", + "format": "fdp", + "size": 6400, + "sha256": "fea33fe3acf805c569b6d4804a257b82275dcd6dba1269fa7ff330091563e563", + "semantics": "NIST ACVP decapsulation tcId 96 (modified ciphertext): implicit rejection." + }, + { + "file": "acvp-modified-key-hash", + "format": "fdp", + "size": 6400, + "sha256": "b937bf76921e91b22e043794ca5b8780b35f60e8384d90faa558ef0e15da9497", + "semantics": "NIST ACVP decapsulation key check tcId 146 (modified H): rejected, secret cleared." + }, + { + "file": "honest-ciphertext-bit-flip", + "format": "fdp", + "size": 6403, + "sha256": "c555ad9f071c42fb7772c778d34a769259a926cae98b853ad5ce4b72f1bbc1a2", + "semantics": "Flip bit 4321 of the honest ciphertext: decapsulation still yields a secret, a different one." + } + ], + "mlkem_backend_diff": [ + { + "file": "honest-round-trip", + "format": "fdp", + "size": 96, + "sha256": "60563d6bf3a00873f259f212eb2916e929f9a2912b6738402345c16a2fc6cb51", + "semantics": "Key generation, encapsulation and decapsulation from fixed seed and coins agree; the all-zero encapsulation key is in range, and the all-zero decapsulation key fails its hash check." + }, + { + "file": "cctv-base-key", + "format": "fdp", + "size": 1664, + "sha256": "fc04cec04516e98b8c4c8c855bc74c30e5d672a04d7be3a44415274c05c1ff4c", + "semantics": "The key shared by the C2SP CCTV modulus vectors, every coefficient in range: accepted." + }, + { + "file": "coefficient-3328", + "format": "fdp", + "size": 1664, + "sha256": "6c274898e6747f79cb4fec2dde374e46f37e578d1e4ee605e0a8ce9c377212b0", + "semantics": "First coefficient q - 1 = 3328, the largest valid value: accepted." + }, + { + "file": "coefficient-3329", + "format": "fdp", + "size": 1664, + "sha256": "9dea64bc7eef0413f8c5bdfea2f74b4cf1acb148ed1bfa846935db0de425eb2e", + "semantics": "First coefficient q = 3329, the smallest invalid value: rejected, outputs cleared." + }, + { + "file": "last-coefficient-4095", + "format": "fdp", + "size": 1664, + "sha256": "465c31c294e5e8a5e0effcedf2e346b2de1643635a66309d7f9856c3fb9a0a42", + "semantics": "Last coefficient of the last polynomial 4095: rejected, outputs cleared." + }, + { + "file": "acvp-valid-decapsulation", + "format": "fdp", + "size": 6400, + "sha256": "22599a5c645de5def70e8e074118f35683d5a324fbc17cd817cc5623c19d5746", + "semantics": "NIST ACVP decapsulation tcId 97 (valid): its key, ciphertext and decapsulation key." + }, + { + "file": "acvp-modified-ciphertext", + "format": "fdp", + "size": 6400, + "sha256": "fea33fe3acf805c569b6d4804a257b82275dcd6dba1269fa7ff330091563e563", + "semantics": "NIST ACVP decapsulation tcId 96 (modified ciphertext): implicit rejection." + }, + { + "file": "acvp-modified-key-hash", + "format": "fdp", + "size": 6400, + "sha256": "b937bf76921e91b22e043794ca5b8780b35f60e8384d90faa558ef0e15da9497", + "semantics": "NIST ACVP decapsulation key check tcId 146 (modified H): rejected, secret cleared." + }, + { + "file": "honest-ciphertext-bit-flip", + "format": "fdp", + "size": 6403, + "sha256": "c555ad9f071c42fb7772c778d34a769259a926cae98b853ad5ce4b72f1bbc1a2", + "semantics": "Flip bit 4321 of the honest ciphertext: decapsulation still yields a secret, a different one." + } + ], "p2mr_script": [ { "file": "legacy-checksigpqc-default-valid", diff --git a/test/fuzz/qbit_corpora/README.md b/test/fuzz/qbit_corpora/README.md index 4bb66bda48..01a509aea3 100644 --- a/test/fuzz/qbit_corpora/README.md +++ b/test/fuzz/qbit_corpora/README.md @@ -9,6 +9,8 @@ upstream `qa-assets` corpus: | `asert_edge_cases` | [`asert_edge_cases/`](asert_edge_cases) | | `asert_math` | [`asert_math/`](asert_math) | | `auxpow` | [`auxpow/`](auxpow) | +| `mlkem` | [`mlkem/`](mlkem) | +| `mlkem_backend_diff` | [`mlkem_backend_diff/`](mlkem_backend_diff) | | `p2mr_script` | [`p2mr_script/`](p2mr_script) | | `pqc` | [`pqc/`](pqc) | @@ -46,6 +48,16 @@ arrays and strings from the front of the input, integers from the back. real keys and signatures that are built once per process, on first use. Tagged inputs with another version byte are ignored. +`mlkem` and `mlkem_backend_diff` share one untagged layout: the key generation +seed `d || z` (64 bytes), encapsulation coins (32), an arbitrary encapsulation +key (1568), ciphertext (1568) and decapsulation key (3168), then, from the back, +whether to flip one bit of the honest ciphertext and which. Fields cut off by +the end of the input are zero. The seeds use the C2SP CCTV modulus key, the +3328/3329 coefficient boundary and NIST ACVP decapsulation vectors from +`src/test/data/mlkem1024_vectors.json`. Both targets run on native code where +the CPU has it, or on portable C with `MLKEM_FORCE_PORTABLE=1`; +`mlkem_backend_diff` also compares every result with forced portable C. + The fixture bodies check that an unmodified fixture verifies. `pqc` also checks that any modified signature, public key or message is rejected. Legacy and generation seeds keep key generation and signing covered in every replay. @@ -63,6 +75,6 @@ from a reused corpus by hand if they are no longer wanted. A target path that is not a directory, or a symlink, directory or other non-regular file at a managed name, is an error; all target paths and managed names are checked before any file is written. -`test/fuzz/test_runner.py --require_qbit_corpus` then fails unless all six +`test/fuzz/test_runner.py --require_qbit_corpus` then fails unless all eight targets are selected, each has at least one regular input file, and the fuzz binary reports replaying them. diff --git a/test/fuzz/qbit_corpora/generate_seeds.py b/test/fuzz/qbit_corpora/generate_seeds.py index 62883464b0..db6942e068 100755 --- a/test/fuzz/qbit_corpora/generate_seeds.py +++ b/test/fuzz/qbit_corpora/generate_seeds.py @@ -557,6 +557,99 @@ def auxpow_cases() -> list[dict]: # --- manifest ---------------------------------------------------------------- +# --- mlkem, mlkem_backend_diff ----------------------------------------------- + +MLKEM_SEED_BYTES = 64 +MLKEM_COINS_BYTES = 32 +MLKEM_EK_BYTES = 1568 +MLKEM_CT_BYTES = 1568 +MLKEM_DK_BYTES = 3168 +MLKEM_Q = 3329 +MLKEM_VECTORS = CORPORA_DIR.parents[2] / "src" / "test" / "data" / "mlkem1024_vectors.json" + + +def mlkem_vectors() -> dict: + return json.loads(MLKEM_VECTORS.read_text(encoding="utf-8")) + + +def mlkem_set_coefficient(ek: bytes, poly: int, index: int, value: int) -> bytes: + """Overwrite one 12-bit coefficient of an encapsulation key (FIPS 203 ByteEncode_12).""" + out = bytearray(ek) + pos = 384 * poly + 3 * (index // 2) + if index % 2 == 0: + out[pos] = value & 0xFF + out[pos + 1] = (out[pos + 1] & 0xF0) | (value >> 8) + else: + out[pos + 1] = (out[pos + 1] & 0x0F) | ((value & 0x0F) << 4) + out[pos + 2] = value >> 4 + return bytes(out) + + +def mlkem_input(*, ek: bytes | None = None, ct: bytes | None = None, dk: bytes | None = None, + flip_bit: int | None = None) -> bytes: + """The shared layout of both ML-KEM targets. + + Front: key generation seed d || z (64 bytes), encapsulation coins (32), an + arbitrary encapsulation key (1568), ciphertext (1568) and decapsulation key + (3168). Back: whether to flip a bit of the honest ciphertext, and which. + The harness zero-fills fields cut off by the end of the input, so a seed + may stop after any field. A bit flip needs every front field present. + """ + enc = FdpEncoder() + enc.raw(pattern(MLKEM_SEED_BYTES, 0x61)) + enc.raw(pattern(MLKEM_COINS_BYTES, 0x71)) + fields = [(ek, MLKEM_EK_BYTES), (ct, MLKEM_CT_BYTES), (dk, MLKEM_DK_BYTES)] + if flip_bit is not None: + fields = [(value if value is not None else bytes(size), size) for value, size in fields] + while fields and fields[-1][0] is None: + fields.pop() + for value, size in fields: + value = value if value is not None else bytes(size) + if len(value) != size: + raise ValueError(f"ML-KEM field of {len(value)} bytes, expected {size}") + enc.raw(value) + if flip_bit is not None: + enc.boolean(True) + enc.integral(flip_bit, SIZE_T, 0, MLKEM_CT_BYTES * 8 - 1) + return enc.build() + + +def mlkem_cases() -> list[dict]: + vectors = mlkem_vectors() + base_ek = bytes.fromhex(vectors["cctv_modulus"]["base_ek"]) + acvp = vectors["acvp"] + decaps = {tc["tcId"]: tc for tc in acvp["decapsulation"]} + valid = next(tc for tc in decaps.values() if tc["reason"] == "valid decapsulation") + modified = next(tc for tc in decaps.values() if tc["reason"] == "modified ciphertext") + bad_dk = next(tc for tc in acvp["decapsulationKeyCheck"] if not tc["testPassed"]) + return [ + case("honest-round-trip", "fdp", mlkem_input(), + "Key generation, encapsulation and decapsulation from fixed seed and coins agree; the all-zero " + "encapsulation key is in range, and the all-zero decapsulation key fails its hash check."), + case("cctv-base-key", "fdp", mlkem_input(ek=base_ek), + "The key shared by the C2SP CCTV modulus vectors, every coefficient in range: accepted."), + case("coefficient-3328", "fdp", mlkem_input(ek=mlkem_set_coefficient(base_ek, 0, 0, MLKEM_Q - 1)), + "First coefficient q - 1 = 3328, the largest valid value: accepted."), + case("coefficient-3329", "fdp", mlkem_input(ek=mlkem_set_coefficient(base_ek, 0, 0, MLKEM_Q)), + "First coefficient q = 3329, the smallest invalid value: rejected, outputs cleared."), + case("last-coefficient-4095", "fdp", mlkem_input(ek=mlkem_set_coefficient(base_ek, 3, 255, (1 << 12) - 1)), + "Last coefficient of the last polynomial 4095: rejected, outputs cleared."), + case("acvp-valid-decapsulation", "fdp", + mlkem_input(ek=bytes.fromhex(valid["dk"])[1536:1536 + MLKEM_EK_BYTES], ct=bytes.fromhex(valid["c"]), + dk=bytes.fromhex(valid["dk"])), + f"NIST ACVP decapsulation tcId {valid['tcId']} (valid): its key, ciphertext and decapsulation key."), + case("acvp-modified-ciphertext", "fdp", + mlkem_input(ek=bytes.fromhex(modified["dk"])[1536:1536 + MLKEM_EK_BYTES], ct=bytes.fromhex(modified["c"]), + dk=bytes.fromhex(modified["dk"])), + f"NIST ACVP decapsulation tcId {modified['tcId']} (modified ciphertext): implicit rejection."), + case("acvp-modified-key-hash", "fdp", + mlkem_input(ct=bytes.fromhex(modified["c"]), dk=bytes.fromhex(bad_dk["dk"])), + f"NIST ACVP decapsulation key check tcId {bad_dk['tcId']} (modified H): rejected, secret cleared."), + case("honest-ciphertext-bit-flip", "fdp", mlkem_input(flip_bit=4321), + "Flip bit 4321 of the honest ciphertext: decapsulation still yields a secret, a different one."), + ] + + def case(name: str, fmt: str, data: bytes, semantics: str) -> dict: return {"name": name, "format": fmt, "data": data, "semantics": semantics} @@ -566,6 +659,8 @@ def case(name: str, fmt: str, data: bytes, semantics: str) -> dict: "asert_edge_cases": asert_edge_cases_cases, "asert_math": asert_math_cases, "auxpow": auxpow_cases, + "mlkem": mlkem_cases, + "mlkem_backend_diff": mlkem_cases, "p2mr_script": p2mr_cases, "pqc": pqc_cases, } diff --git a/test/fuzz/qbit_corpora/mlkem/acvp-modified-ciphertext b/test/fuzz/qbit_corpora/mlkem/acvp-modified-ciphertext new file mode 100644 index 0000000000000000000000000000000000000000..1846c14ca16955f2c6d493906518e26e316ecd1a GIT binary patch literal 6400 zcmeI$cOw*z!vJuz&gSgB_sC^)#o1?%GtSC7n>c%R_Bh)aXYb6y*`wmjvk4VR%1nw- zq53^9;CTt(H}U!7<5IcFW|hB44NbulJ&5YN00p!j3cFTqv0E1`(HNwT6F-dZzr6pr z?MTF}`aOpYW|`J7ZGuESX5dOLu>DxnJ?PmNF+}YLE_l%@y>Z4A86#L91UovGiKR3NwT6kNVIi6o@qmbKamuIw0G{}HeaiR9xhz-8&85%Zo&a=Q_ z0;J*IBaOB`(IO+7Nf$uxZTzUvEjYNk&O1J<6v_^Aj=Gt)he61+f7gr5K-+RJHEW%N$J)7E*(k>{Yb*CcN~FBevo~ zQzV!jroiXb1=CSQ2=&oUv7Am>LOnAPj7J{nz10W^w#F z!-9pC7<_7xP}XrKyi4F5o%8&5Q7L-t3kWCbM}jL}vD#@GGe_uTQ7sU~*xD)rBRi%7 zT&o|lhxenb1dHxdgptwYS#2KFKa+UI0g@!99+$<@&sS#;?dR^)kgOO^0J`dUU08W; z!KfY5Og>$VVz?gy@tCc&SXkSleD2d**3%Ox@Mq*FExU9gA~GSb->MII>Ge=6kFQ%T zTjXBU)X{@g&71su;#kqW4)Hoy^*lHJgyhGclcFX<^!zuxMCn1hhfhiP2yM+Wql!S9 zovLflEBRNoT3__T1X4P?__J4VMpn$o-j^X96*L*3VU~zHQR>Sog|k)#!d>MmPcy?y z@F#@87;K~OIZHhKWUQ{Ssm(OZ=Mf>yaPk(MQdr}S|(Yz6u+xzMce9e)>E3twRRbqCrX zaQRE+Pz%qgu0>iB1IRwo?K@qk`X<;Y~n5#b#qDk#{e?ijTjqDmbqENd*zVVP?0zg=dG zD7ISzvmiZi6PFLvPWRb4l77wlde{*t#?`o_cEaO&;M$W8Eyr&U1V}KjdpsE6@R7|H zbgmX+oHJ$nN>g<~6nUcR@A;dw{nXguxjh8))|((8taLh0-}(=HFkhSBb#?#Bjw|?Z zLA!S&offE)+Vm>tTs>F^bRTo$`!HKs<~h!yFh!}cng^ir9>{h!+{_YzQC2&+GH<1( z4Iz}Vf2gt9OtIb@SB(AY`mp4R0c$q$SucjQ^ygYiznS2LuNEKcdfOcs>zRXnOL zH*+le*!>W|KFdt;7bYw7kn{__>;pB?Q!51J4;RXxiKhSlUhdca?Dh9@GqL9kJJ%Pn z=u6&0nL`a1Od%=JEq71TX1r+G87l{d&BgxM$!8OvN(4pVy2_R&*C~8r@#GcND!b2m zjUBfcU@t!`KHH!5n}27fGHHVUsw%aXGybz)AiZ!_h>Ek0>qSG^mHtC|YGszecLxwt z+7NB?f4tRs(hihD$j-i3tsgo=HI2J7dVjt!?hlH{xP&^dXp=h9s$bvRTx0?Z5OEvF zfHy;ejrIFvC2)JPWAh-E|CnM$vZ6_hIci=$^AUk=3xBe3d_cJ zoFAv)*r%8_&>l9915Yn)I)-n+4T6i2Fy6&~HqTh@J+k`CzopmcU)x$D_~7tV*4FLO z-Vcyd^i5M-)`}Q~tPq9H5pqs^t|r%6s{$0mw(y9AhK~M% zFE{-31TW^U5D zm(BPG*RILDii$&Hb5`2hNKITS@0#L@8~s0(XB3rm4X!tg9gv(qe^gRzgi8}7bg~uG zx}~A~tY-n@ym<;9i|cb8#jnO4WMC|PtATL^e&vtd`t^9Eq@-J?53_h6QD{g}l{{eh`uevRsQ}EB?=$!7*5^;WRccaS`G0;5QjLLUHDG>f9 z;y2q>o=ARZsYX{jxPqAHk4s&F6{5U?v2u$*4_ppVYNHNO=vxX{XK?o$Y2sKn`d%b& zT9#{>a!?Dh@HcWi@(@vrNx;SouJVlm-EHr01n~I*uN&{HEJM#05PB27kW+_P zqI#WDL)+eq{*Eg!ATN91@lfH#R`HYJp%dIPwRFbrJepmjDCg$_xyjSh5!(2DJ0+=k z#)lMXo?j284<@p(M}Te?TV0(XT1jHnq^eN|J}f9N^JT{^-Z7Qn2C`va%sM2oPB;_n ztryFlglT2^hgwL)QK2r&hO}F=ZhMr3i{X@g?#9bsL;lZ%AL-q!CA^<~MO6@hu5`q( zwJBO>J!|_b*ASJe+3~VQ>W_iFH?2!|hn0o1fN^+j#t-8cJUczmk9U5?uoP{6HwD#h z)qC>*#oaLEu<`S-XD=4l++!pHJ-;5$WbL|)rcqVO)H~{ z*+e_IBuKljUl=mqMu=myE9|ltlO@1qcR=^Eq>pgf;6{OV0sxwH? zgFB?(jFxVw1=-=SRB=UlGka5>OA)jtvLaioLYN_wH^B@mfFc_fKnFs=Pztcx2TP2h z%6K#179QHfoI%&DjGRGUkY#;5nrZs<0Im|3^)XGD!!>14YX97J`B!sg#JKxq*zF-D zw%k4Rgmf4l$n?%R>3k-xKeg~Yb5m67NhhyoarDPu2&7~(UWr>re*V3R?|P>|$%R-{ zOiL3=$Ox==R>|`Pz-S<}uTa#nio4xsJPZxXkjR!21tQh~9pwBDpfW*mq1B8)J#g zEg++Ne1rOPH8SU3#j7&BGRY{wX^}?y)QM%t9(^ZnFdPw_@!dVPhQ$BalXdnpgKt;X zzq~OaAutD0ff#`&PHNBPzzADr3ELz8BpPKC<<63%kNvFtiH#X+fs4JiWJlxkuPhAG zdY*X>_;9f%h&&-PpsapV%^)Y$ziPteHwvw9tQ#oeuLHy<3jC1^_|Wh98R|6iR?={a zE}E5+MhthN=;T-}6=$ZTCJPtJ(5j@2qEbb8xkWfUg&sgk%a{vK#ri%Z=I}~BC#gns z3Q4iPiwv2NliWZWg}a-i!K!+YgX!+Y*vOuzxm@3E1-l25KjYRwV=R1LRiqxeYhs&o z-R}0tlYF|ROHB`^VcjD4`i#n*o~gGD{O+Gvz~UIlNA4BfU@tKE~ty~ zw%ApXOufL+rCleP`~-?Fx6r0aPX0N%mu@6- zm<$LNTEE{cucEGX~jNl)~lthbO}9<-hqEL;S#RVOqZj@73xf zDi5M+iA$dNIE_5bk`0A7g%qi=i?X8?gv>MMJ=o4y?5+(N#&Jb63>@q#Syp!|ulQh1 zK>U+K?r!Txo}}o?cTt9Jy&A%z#7R)r!?|cRo^7oMq}H{sj^m_%j444AK#fk>si}q= zNvU5(7f{r(oYs1N0+i^*L2Gm^RX7-{S|Te2FqEvdC?+d`DA=+5qsY{x6yQsYZk#UD zR`87{4+|q@5!Ti1XsBm})nuH-Vv`{n)NM4=vh77I7iSR~%?r}rxS_+v3g-?^qPDo{ z&K>lFG#`|qf$6__>A2Fr&;=6&iW@pFEZg4Z^W$$73m1ytvXJ&j@-pnd3eYaiW zjs*V9#~uhh>~tU{Yke%ApckXvc#U@KE502R% V!J1w?{nlQ-!5bJUr8m3N{{#Do>a+j= literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/acvp-modified-key-hash b/test/fuzz/qbit_corpora/mlkem/acvp-modified-key-hash new file mode 100644 index 0000000000000000000000000000000000000000..863f876e39f5d730718019864cdfc7d1c258f268 GIT binary patch literal 6400 zcmeIyg<2Dg!v$~{-8GQzW=uwdGz_GqdmuF$X=LQ6!RSuO(V=v!bPSLbDFs1VN+}iP z`(MC!3Gcat-*e7!DBWN*%bFwCMvdSohIU^;UNj!^IhJiQnP<;a=p>BdDuwl2i~BSk z@jI1&X12&#pwvwq11N|0-bna19}779y*?3yS8lOd!tIe$Er(qzGJ+KqEjGBxGw#B!f$ z87~Lw8A5VopB&s6HIh>XVvCu!Dl9fo=6j>k5vPu~9qNzOuloQ$a|GBBmraVJt=LmT ze;KiMneT>TkeE}UP)tV3aN4o+Au-c5J;^^)F|ZQe$w=`QIZmw^ob;yy>20jR!@ukO z%G)k)H>YPdEWtmX@*8)@Zz7Gfbw3!WqQV-8&Jw5GD4FO=dk2gJ|5{6?;hqWmg|xR7 z&yTN=ctng4l$6WuyzbJs{X%2h?Z`e1B7pxd)5a43D3i{o~ry zs`0LDEZ|W*JQK5YdcOA)Vi)$OE;4mdkVK4^1bXB@BQ#TyVXs~S2?u&V1f{peuE5n! zaele-V85d*tNErq z%us0!5PYe;odR3qpF*7$_J0*a7tn?gDYdliqKM^5i7;328e*gyl3fir*F8(s3bONwWC z>VU8uF3ATG!;Or%6*r5~EyuR;%#xBreWOQ|_XxH21dfl=i%!%Ja<55Bsj40S(6y+h z|NSe6vfwL<5r(EoCw7Qxb3M9zA;gg>_p3Im%mbT9cvX8#+E0Gm^Z| z+Q_n{fwM;pwP=?znxJfaQxO^tE)~OeUwrNtU%C|wrPfKmH(subA3wYjC!L2pymfU7 zAyJ@;SCbBSf&(^l^46%{D4h+k#PVX!gJsY0IyzSmO3od)IliEU5>OQPr@o?k6(&wk z?`I2tTMpr|W$S%`W}CzXww+A!CNGa7L`0lgJ?Mpeae@N#$|SMV$j$s1m8l>ryrbJ( zW~_a(+v6J2j5JR?OBE{`{X~avQ7A?au z)g5M&{mgks=U%hIBRbijjaHQ;2+zP2RduKma$V94-&;6Y^5r`FA_C1&UFO4ggO-@U z>I2;rmMwK1J1or4DL|A|!neJeRuR)E+AIW7ior45>6~3A`TS7{ymhigo!|bA^&d@O zNN@dgJu=GMD zD>TktZ|j&>^e*xw4T>{NPzRL|V{bj%BNzTB;W35xp_A)7GmJ^Y1UIC>2I>?!HTxV2mSU%nM-x3m7jY2g1p;U~J| zQ31y{PXReNaT^*wU~Z5MT}l0N%GysRZ+N_*lJK*4Z?$orXh4=Pl{RujgKlf|iha9N z`zzsXI78mXg#n~;v&x;_Z}si>s~`Ss1z#t7_0k!O!@tvZ6|g$LBOuwY9KT7>un5p<4_p7Eb8=gatfdR1P9`=Ky&p&($@T9yx zG;$l*jr0!;++)eENo}M3y*2wBVUPNc+Ep_^_@J_t@o%-ptL%daK?BJ z5n7J^0xsXZMyReyb5e;cpbBR}KFMDviKWa(dxU}@FZUy7UjWAUch7|y#!qJy$Irk; zJdJ!g;H!;agX2qSnYW{D#eK{~XY>OBvBgouy(i1l(u-=53Nu;!K1?UIT>()umT}1! zZyssVRYANF5^#?8JU(qNer!I$x8w=}{1-{w-mTH5^5py-W>5JUH3^Z|+*6OGv@d!> zftj{cmoU$26v)hh8^Wx=LDb2cqO_KK)r6dIJk-~zC7RKcD*erbSCId<7E(4j?1-?^8ji-r??v)7#T}aVSz#crUhHSs zNzae8bg=!AQ`il`_u0uJ2{%sb_wQxl9G4k`EM>nRRWx-M{v@b&!=v3~@Cg@RD zy6mi71hmXlu=IG`m|^IN%-?nx?56HY8)#aUQV5?RT?(Z<7q6P774Ir~^P#BPku3Mg z9(ck^-|ZtJ_=H7v#x*29j#N{@e`M~FhV?g2;q%`*k=p`Bk_;f-kvfqkJe0oNl#2H) zcpt#Hsz6D(Nf=6>wI-jA5sAab4;U@jJoG)f{iFhvCrm_0i*6ITm(NAslH02lZYp#> z1}K?dYwYgA?(^kpbh5Jso7u7=94|`pRMQo+UT}x=Z-ZKQhRD#n0TmzVmtHacu;K#4 zfVHkk^1&R8-x3{OX)adMX9!U3(KL0B^h1A*7w${ttrN!tS^@J*O-_G+0BF)1>y zReJnr=3$XU1rBMKy%N%!Ja2mh8}Xo|U92GeLg` z<(K;&I~ap{uHh}l;yd@$I@_fc=IFF!QAN_IKbN^j3pG3k@^^mG9c$u^e8mwWvcLnb z<-%MF#%Jc2B-1p6M_&mA6T6leZAVBOa)mS}S$uk)7`_ZhhzgH82ZDHvRd+Jj{AZGU zo-#A}GnXcW?K-)4+yM6q+>x;v_6|h)_GVi$eAHv=(3I)5OXBE7l1EN?r<*-A$x7DT z_2_CZ*GvJfwhUBK(9J3*QkZJ`g7l2eNN%+Pl>q%aEJ}D`@pOqN!6@NDC{0v`mvl|( zwhl9`$q(l&*}p6#7K|l@YrGKF6U$BBZ4fnJ5_OeNE7bOlGgalT_WXA9-t!Z<0R7Cy z-NS=vrgCVatUg#)PujOf6DB;pp*hr<$YJwa66yLp_S(dpa9U1IvStH=?=C0|5$Nm= z`{bK#NW$>|Rk*u#^1ah)tn7}4kSKraDqK=~$4yCWYWdZ$CAx=Wb_urCWdD4?Am3tG zKXgp(h_dR4&&EdRHV)A!{f1>FbDp44h1Pu_J9nz!#abn9q%33WUBs_AANGohw65s# zJ%?XE5nMW6-_#{%$3~aY>K>McZT{LN;taObGs*Cmc`);SGd$Jkflrgb#Qw6@%~ZM} z6F1-Y%cyiZuK{f`$BBH^wt&WZ-#}4c+6FH8aDJUeqn?1Ox_#&D#ZC#Na~280^Z<)- z)_HDs=xU{L!$nFGHHW}Dwyh!-hY+{bm3McbJ^(I5P)dvEP0nvddmL!;y1M03z$M3O@W+qv9BQQT2~luhSV?5L$^w6=O4RO= zCY!L7$nmlYJ1CV@T+HWw;xl6z#h4l$l<-wfc|cR$5v zjN)!ch|{E1ElbuKP1GF`ahX;Qxkav>gu0=tW2dA4>#s2rOx0R(8b5OBCagQ2DHsr1 z(f|Xn?Q#-hJB7vbthUpBmv@i7s>4dVWV~ckI0>~ib{mjVfq$dDXo?_QDl}nLS(qhG zsHy~v7}TsmMaDf~R%0g;4_0c$2hVFB)_0q|l;oS@T)g5ZZ!2Nz_DCp}7JM>SXO)h~ z2CTz?V@-HGq!MYnliDGL^`LjoHj1U0+GQFX4aPL9evSu@j&BjBBxc1zI`}3Rf|v=J zJmH$&ypm*Sz2w9k_dWQU)rDS_lQZ5-x8J(5jcCE4)+PjkLzZL$3Lcv0_=&ON5_ylH z3sZ|N;tIpb2QUd|X;JAaR@KWz8hz-xiydP~z2qkHs*N2S?tgE=kKE%=`BTM?lKLk_z9Y5L^$%2k{y3&T^N9% zr3>#<{>v}kL)=XEJ3Rta)Ul%moup$PV+f98B8Y7^4&{3r>;5+|O2|0HK``=cEy8R1 zJ%V|%p0v06YnjDUVf1eOmsdD5{RMH8RN)hDWN?G87_2n|KBQt4R0Vw#s&7A2ds7)2 zmifxtw?zzKi;`qgNBifku#m?AYx1U^`$~kHzMa=2#{3!k6>C#OjGtE_mWEfC7M!mT zU%yU{r_u`(;!gegM^SIhBMUz+DGM`2+wwN?(G3^3+Pv34q*7*=xeK#VO+_4A3l&Ea zz$hu7!-1(R+Ycn>)vGZkqFIc8;DkP)S4VrOk|1!3gv&@~EC*3BIkn)r@`t=CWm6@) zyapyo)8|1ZG4W-2{PaDYnjh_pt>_~w?0$jG0{#R)wZmA4R?}b9Hc&OA0%qBNT zbLk?`8iHxS6=0I2qAL_lZfcD-sZ+EPvKh@kADavsH5E|@#9tfiK+WQ1TVeNjT?Oc! zV-|KAEP4Sx^?gVnMbs`KC1yy``cJs%afzQgf_E*#JsLUK#lIh0+57}+ZalRMVBQiJ zc{}a>Dp-`1R)Bz@j%If_s?{SW8~>#lgW{dw_GAmTqaA!19->P&RFNg*O2fsC5aF!M z42>GKXE1Z2QFRGwd}aOM?t@%Th8NB3DBsaf5@^*2g9cY&%6zz=g;=Uxvb1!-bnRyy zqyeA$?6`$V44qUjC>x%|JCZa>@$Ep}-_>ikJPERv&KIJ&wijGJR_1Co31f6jsL1@S RLWo{r93CDfAIv<5{~ub%>|Ou> literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/acvp-valid-decapsulation b/test/fuzz/qbit_corpora/mlkem/acvp-valid-decapsulation new file mode 100644 index 0000000000000000000000000000000000000000..6fe5560bae189a5f8255ea549ea143776a9c6b64 GIT binary patch literal 6400 zcmeI$1wR~&;{b3@$8?wS+=+8cH68!pW{=4#XZeXfauQc1qVEpkgE`}e zmG3iK=dMr~BuxQSBm3_qeVfk(T>M%t1YIjXvfAdYQ9VP=;HyUs{*d->y#%>c9kAKu zujhFhmnGJk=UQb2Opr7gd_NNrz2G`s5s=(rGY#N#Bpr=u>K)T8$CVD*Z7A&Y<-XoD zPo!;ARI~L?oj(&cn@@KBjmnkSLgl)q{LS^fGcTl5`DYEu%h+w!o1Td`u>Rm8WsaF>yU#=UR@Y56SYJWKA8iFU?=$ z=Zp!qpu(YWNnpCIW|=fyx2lY!<>gj~aZek&bdK>;rCkA)+1loNSIO~{kY-q!pSuSO zrh1|)H$uleWa4={jwYsyb3L|sg{mrpG?SF46ZtE2tIlf+aG&!EbEv1g}sd> zwFrz(PZ-+Ri{v6v1i{f)*&LzfK$XL{pBoxJ9xR~u&@#zU9tDC7cZ*HpWZ*Be=C3oerabj~opXwZN#amsA-&)yER=VS~S|03*WB4e8|V9}02 z^X&+cP&h?AsSG`Ts2h+$78LNwdXv!d`fU8;Pm8BrGp4}A%@s|;kP6}J*NP!5l$$;$ zRI$B>uC1~QC(8>Del*Vz^oKaL-ppym^yFCs!_%?jzqS#QNotV#w>mroAMga`?-fYM z#g4k+f7s(ilrgNCz%1|6Zhk{nxFl%y_G!GltM0gp|t3)0N1J0tbn z7GT||6>YmqT%B+W^Z1lrwwqLbX8$CAs>zF?m%)PWv$`c1xUi1dC%syCKgh6E`_J)S zBODYmRcrW6Nba#c8!;qi_Ym)=yS;w~f^qL=rPQT=$o-YZX ztZjB3QXJb5)u=J(Qj|xkVbHY3_ER>WKPKvs*Aj9t%*O}AOvua!s(K{JU%5FNfz)|| zeI-5vc(GxjaDp3;wi)ga$YuA7%msu-TZSvY|Dd{-e-nuTBqnT|TTj(fRfX_3Ff+QPp_% z6x%7rzU)cs`X7URgiR8Cy%RrVBf=`_aD)$7TMC1a4&ah$rQYV$0^nTj~1r=|(NmIwi5 zMTz3RHhlq(s!eu<)d?^LTv>6O1kTcV2OWrMNML$skl^#Nk|u71W6oD>E#+{FLd8ok%iL|A+}x9#~!`5|vqX#Hw=!0AY3wztSF#e1Bu)Tvi*(Ozg$ zuw465)D1}!akipI&8140CjZry#-z;D$wIj}Ao~4i;Clg;f9{XIw4sTqynx5}e@?Ih zs}Ae$#DaTeDQyP5i3&W@`YhkVeVHEZOTj%D`8at=2W5k5*VaPAEFE?7CiCEJ+7BA^#{HvXRQM&imwHQPC&pvlCL73GxGxi&Yw%y+;eoL(f%IR zOe}kS`+bRx1<~TXd*%>*zhe{Gs7z+`!?mlplded!;YUd?3^^jBNFKd2cl)j}0Z4z8 zDHu7Iczw(32FQis`!?Xi-**ex43q_0-(#6^mi=?D0eYko9W-~N{>+g}) zGwvCW-we6myM2g|&@mUv;To$|OT>*=7!ftm`K`|+iXa=1lP^;YfH#t<_b2(t`$3!8 zCb;^A15DK_W8zE>((bIg>DPE1rMuOBRrR{-F3U!kzT$mkHPyQpvG~qzarL%xv^4^Z z?c&^Bv~`K#C>edi@MSr}=feVr2KS6T(bdhh5*)-BSeawYas=jYXs&oZ zvionVYkre%I9i*&lp`ZTR|A1j_L+a>{0~-sfH7j6h(+%@?aYx$;Wk@qsB$ELlgEOt z?Cuz%@FXwI&%UXdsOK*?v2oz+UTvcasoF=ww1imtQ@4 z!V12zj)goZg(A>o*bT>}#ZR_C5M5o#n<7wB>G%?Tw#(G2XaA5;UQESmyh&z)1Dvu{ z@5@r3=oTmCFODs!*XH!lhvqsHzB@;tF=V`pqLCWvg_OqKL#eJTT^pPt`A>k?0p2;= zQz-WyQF}C4p8xL!%LzJt;C%DD3zo>^vj0mMbAadZUr zDhR)}n}*Jj(MxC!=Es-{yey_!m0W2jF#l}^mCpFyvZjb@Sy%e%3G5Zil-$Pq*vR;= z8+44sF1aky0u}f4D=`Utsej`mJ_eXQ$$b|-0I5ci#!R@yY);H>DaN45*|u9M)jWe7 zy>_f3Or>F_#$|qEHloGo2XBJU1SO)MjwGTS+=1-v;i9gw!f@Wzi@&gAmr>8RYAewG zf8+f&*ifHgJfh95r`>){pQvDe*h@CE9MA#lJQ98sjOShB@By^BupKHgvf~oZMwiw& zEzaqz&gpHCJ*2U2XlcZbb4MPt7H9QRXaYC*lngb$0{UP@Ec^Q7$x{7i#1%^x>rgEf z2T@{e7)uY}`ID0Dw+Yq!bK`5N@iEX&&eW$E!k@dh>oR$c^Gj>oLL|zSw0~Wgp(<;; zwaghUlZ<1})Rq?bWhy|pFr^FSQ4qzb{fqFm=D4(I*=JZ6KJ5bIM=7LDZx6B5c-_d9LSoIgT2P~ z39|A2Pk!^z-1lB~`Fs@0LyJ>1&69IufkPSS8!#A1x?xYMMo{k=Hj>a4d5JNmoPF25 z4Z7u?b;BfG^fM9u>xm%pQ%EEFJpU4qWo3LQFy$eGE{Je%XGW-LB465tT&Tkymn&=J z>hN}6urlPe(;DsHDteFEB-=TQbXRZZERY<@ThI8g^L7X5rm|>gJj#{gqT3%&Ypn|e zb%c9gmw4bqoQVO^3|nuXZ~1?Ssa`#aR+_+Jt}T0}%M^Peag3*+=td&OWvXM6@|RH- z+{N3OukTn)W*Cf}#)(tNLIWz)HSn5wQyTv@E%U;2iN?-?6^8i3e00=(=DHn>zbm@P zOS{a#g+_g)80YdPM&|O-LC^qs$n5zs`762??X_Zqw`K1t|U>21mDvX!OrJx z>o^1PlN^l$k^C}Tp}t)lV{x@B6(sMU#^Z;B*8K0zr?Nn}{&?-fL2f_0flVouqBVU# zcn3cq8Sn3GKZg}?Y@j&yx}$`H2FPh2glv;Tu(whQxps=Qc2dK)dMWgk4vZo6fhe6_d-4ViylJzJRU+B;rg!G#&Z|Biw2@MVj(7mUF&A%9 zt3WzJIh+o-hJD)saeu0e?-QVB3lVGq4+pd`ojTiUNJVlt>8zkgZHyO zLXFqH0XGR@HLQdjLPz6S6r;7cm;x>(^HEBzmy*DqnBnP^8VgN{3ptI5#&Nfv2sb%3 zcXoR6!t9!EZqul|hF6^_ylnMw%&JT<#Gm@s$qF~`b|{IJo95a^oM6hFJk+S!wCZd| z+q#&*TitSLhi)2*DvJOBtW<1=h>ceUYN35qC`|p)Qej>xIj=$z8uBEjDmQ^XPF6*v%EfzI-vvs4j(nlWaN5 zx`@j;1&C;Bkwg%V{xRb%XR_p@^3AoWsiZ#6IA!hm z^#OEo2Zdc&QXvhby%iR7%`+3{jk@)UDNi-AF-!`fKN_++*G>N*5&7bRa?)B6`U|ud zW|!$P#nZX?lHu)NPc4e+Y(ZduK)h<;>jvAC&i>fN(F&P$8#>U8J(PO92s9^Sw ziw23OIU2$P_CLpv$-Hpx#<2eycTI$<4zaJBh{gp@j4|+kJ7D|$d};jzJ|F4vc0K1$ zIV$JU8J=pR7OpMjrCiB>fArrU{r5-z{n3Aa^xq%- ze}7b+qv;e=U$$bJxe;gYqx!73Pf&$fsL*Kf0UFl)o1ao9 TaT7I*^+w-KGY6D%?xy|^oqFqc literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/cctv-base-key b/test/fuzz/qbit_corpora/mlkem/cctv-base-key new file mode 100644 index 0000000000000000000000000000000000000000..f4eb8e847187176031bccc800c98f954df4e8137 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoARom;=uM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/coefficient-3328 b/test/fuzz/qbit_corpora/mlkem/coefficient-3328 new file mode 100644 index 0000000000000000000000000000000000000000..21a0a6fa803542c309e985ea41a5bc211ff77070 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR0MFX7x literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/coefficient-3329 b/test/fuzz/qbit_corpora/mlkem/coefficient-3329 new file mode 100644 index 0000000000000000000000000000000000000000..037364c402af2300a1fc628a5b5bf506490e915b GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR6Wdoi7 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem/honest-ciphertext-bit-flip b/test/fuzz/qbit_corpora/mlkem/honest-ciphertext-bit-flip new file mode 100644 index 0000000000000000000000000000000000000000..e90fa0ac39fb8f88ac32e96c5e5c7600b649e4b6 GIT binary patch literal 6403 zcmYeCcUvNK(s>@ww!;jzRolN9=B<9Anl%58WZ2{jyk6}`811Ted^O5n^HME&!Ck5F zDHr*?JB~8h*X;aeT(I_)dP?EC*BYsd?#o0@yDZ?_b)3bqZubwTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355z4ZS|2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR`T?DiM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-ciphertext b/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-ciphertext new file mode 100644 index 0000000000000000000000000000000000000000..1846c14ca16955f2c6d493906518e26e316ecd1a GIT binary patch literal 6400 zcmeI$cOw*z!vJuz&gSgB_sC^)#o1?%GtSC7n>c%R_Bh)aXYb6y*`wmjvk4VR%1nw- zq53^9;CTt(H}U!7<5IcFW|hB44NbulJ&5YN00p!j3cFTqv0E1`(HNwT6F-dZzr6pr z?MTF}`aOpYW|`J7ZGuESX5dOLu>DxnJ?PmNF+}YLE_l%@y>Z4A86#L91UovGiKR3NwT6kNVIi6o@qmbKamuIw0G{}HeaiR9xhz-8&85%Zo&a=Q_ z0;J*IBaOB`(IO+7Nf$uxZTzUvEjYNk&O1J<6v_^Aj=Gt)he61+f7gr5K-+RJHEW%N$J)7E*(k>{Yb*CcN~FBevo~ zQzV!jroiXb1=CSQ2=&oUv7Am>LOnAPj7J{nz10W^w#F z!-9pC7<_7xP}XrKyi4F5o%8&5Q7L-t3kWCbM}jL}vD#@GGe_uTQ7sU~*xD)rBRi%7 zT&o|lhxenb1dHxdgptwYS#2KFKa+UI0g@!99+$<@&sS#;?dR^)kgOO^0J`dUU08W; z!KfY5Og>$VVz?gy@tCc&SXkSleD2d**3%Ox@Mq*FExU9gA~GSb->MII>Ge=6kFQ%T zTjXBU)X{@g&71su;#kqW4)Hoy^*lHJgyhGclcFX<^!zuxMCn1hhfhiP2yM+Wql!S9 zovLflEBRNoT3__T1X4P?__J4VMpn$o-j^X96*L*3VU~zHQR>Sog|k)#!d>MmPcy?y z@F#@87;K~OIZHhKWUQ{Ssm(OZ=Mf>yaPk(MQdr}S|(Yz6u+xzMce9e)>E3twRRbqCrX zaQRE+Pz%qgu0>iB1IRwo?K@qk`X<;Y~n5#b#qDk#{e?ijTjqDmbqENd*zVVP?0zg=dG zD7ISzvmiZi6PFLvPWRb4l77wlde{*t#?`o_cEaO&;M$W8Eyr&U1V}KjdpsE6@R7|H zbgmX+oHJ$nN>g<~6nUcR@A;dw{nXguxjh8))|((8taLh0-}(=HFkhSBb#?#Bjw|?Z zLA!S&offE)+Vm>tTs>F^bRTo$`!HKs<~h!yFh!}cng^ir9>{h!+{_YzQC2&+GH<1( z4Iz}Vf2gt9OtIb@SB(AY`mp4R0c$q$SucjQ^ygYiznS2LuNEKcdfOcs>zRXnOL zH*+le*!>W|KFdt;7bYw7kn{__>;pB?Q!51J4;RXxiKhSlUhdca?Dh9@GqL9kJJ%Pn z=u6&0nL`a1Od%=JEq71TX1r+G87l{d&BgxM$!8OvN(4pVy2_R&*C~8r@#GcND!b2m zjUBfcU@t!`KHH!5n}27fGHHVUsw%aXGybz)AiZ!_h>Ek0>qSG^mHtC|YGszecLxwt z+7NB?f4tRs(hihD$j-i3tsgo=HI2J7dVjt!?hlH{xP&^dXp=h9s$bvRTx0?Z5OEvF zfHy;ejrIFvC2)JPWAh-E|CnM$vZ6_hIci=$^AUk=3xBe3d_cJ zoFAv)*r%8_&>l9915Yn)I)-n+4T6i2Fy6&~HqTh@J+k`CzopmcU)x$D_~7tV*4FLO z-Vcyd^i5M-)`}Q~tPq9H5pqs^t|r%6s{$0mw(y9AhK~M% zFE{-31TW^U5D zm(BPG*RILDii$&Hb5`2hNKITS@0#L@8~s0(XB3rm4X!tg9gv(qe^gRzgi8}7bg~uG zx}~A~tY-n@ym<;9i|cb8#jnO4WMC|PtATL^e&vtd`t^9Eq@-J?53_h6QD{g}l{{eh`uevRsQ}EB?=$!7*5^;WRccaS`G0;5QjLLUHDG>f9 z;y2q>o=ARZsYX{jxPqAHk4s&F6{5U?v2u$*4_ppVYNHNO=vxX{XK?o$Y2sKn`d%b& zT9#{>a!?Dh@HcWi@(@vrNx;SouJVlm-EHr01n~I*uN&{HEJM#05PB27kW+_P zqI#WDL)+eq{*Eg!ATN91@lfH#R`HYJp%dIPwRFbrJepmjDCg$_xyjSh5!(2DJ0+=k z#)lMXo?j284<@p(M}Te?TV0(XT1jHnq^eN|J}f9N^JT{^-Z7Qn2C`va%sM2oPB;_n ztryFlglT2^hgwL)QK2r&hO}F=ZhMr3i{X@g?#9bsL;lZ%AL-q!CA^<~MO6@hu5`q( zwJBO>J!|_b*ASJe+3~VQ>W_iFH?2!|hn0o1fN^+j#t-8cJUczmk9U5?uoP{6HwD#h z)qC>*#oaLEu<`S-XD=4l++!pHJ-;5$WbL|)rcqVO)H~{ z*+e_IBuKljUl=mqMu=myE9|ltlO@1qcR=^Eq>pgf;6{OV0sxwH? zgFB?(jFxVw1=-=SRB=UlGka5>OA)jtvLaioLYN_wH^B@mfFc_fKnFs=Pztcx2TP2h z%6K#179QHfoI%&DjGRGUkY#;5nrZs<0Im|3^)XGD!!>14YX97J`B!sg#JKxq*zF-D zw%k4Rgmf4l$n?%R>3k-xKeg~Yb5m67NhhyoarDPu2&7~(UWr>re*V3R?|P>|$%R-{ zOiL3=$Ox==R>|`Pz-S<}uTa#nio4xsJPZxXkjR!21tQh~9pwBDpfW*mq1B8)J#g zEg++Ne1rOPH8SU3#j7&BGRY{wX^}?y)QM%t9(^ZnFdPw_@!dVPhQ$BalXdnpgKt;X zzq~OaAutD0ff#`&PHNBPzzADr3ELz8BpPKC<<63%kNvFtiH#X+fs4JiWJlxkuPhAG zdY*X>_;9f%h&&-PpsapV%^)Y$ziPteHwvw9tQ#oeuLHy<3jC1^_|Wh98R|6iR?={a zE}E5+MhthN=;T-}6=$ZTCJPtJ(5j@2qEbb8xkWfUg&sgk%a{vK#ri%Z=I}~BC#gns z3Q4iPiwv2NliWZWg}a-i!K!+YgX!+Y*vOuzxm@3E1-l25KjYRwV=R1LRiqxeYhs&o z-R}0tlYF|ROHB`^VcjD4`i#n*o~gGD{O+Gvz~UIlNA4BfU@tKE~ty~ zw%ApXOufL+rCleP`~-?Fx6r0aPX0N%mu@6- zm<$LNTEE{cucEGX~jNl)~lthbO}9<-hqEL;S#RVOqZj@73xf zDi5M+iA$dNIE_5bk`0A7g%qi=i?X8?gv>MMJ=o4y?5+(N#&Jb63>@q#Syp!|ulQh1 zK>U+K?r!Txo}}o?cTt9Jy&A%z#7R)r!?|cRo^7oMq}H{sj^m_%j444AK#fk>si}q= zNvU5(7f{r(oYs1N0+i^*L2Gm^RX7-{S|Te2FqEvdC?+d`DA=+5qsY{x6yQsYZk#UD zR`87{4+|q@5!Ti1XsBm})nuH-Vv`{n)NM4=vh77I7iSR~%?r}rxS_+v3g-?^qPDo{ z&K>lFG#`|qf$6__>A2Fr&;=6&iW@pFEZg4Z^W$$73m1ytvXJ&j@-pnd3eYaiW zjs*V9#~uhh>~tU{Yke%ApckXvc#U@KE502R% V!J1w?{nlQ-!5bJUr8m3N{{#Do>a+j= literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-key-hash b/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-modified-key-hash new file mode 100644 index 0000000000000000000000000000000000000000..863f876e39f5d730718019864cdfc7d1c258f268 GIT binary patch literal 6400 zcmeIyg<2Dg!v$~{-8GQzW=uwdGz_GqdmuF$X=LQ6!RSuO(V=v!bPSLbDFs1VN+}iP z`(MC!3Gcat-*e7!DBWN*%bFwCMvdSohIU^;UNj!^IhJiQnP<;a=p>BdDuwl2i~BSk z@jI1&X12&#pwvwq11N|0-bna19}779y*?3yS8lOd!tIe$Er(qzGJ+KqEjGBxGw#B!f$ z87~Lw8A5VopB&s6HIh>XVvCu!Dl9fo=6j>k5vPu~9qNzOuloQ$a|GBBmraVJt=LmT ze;KiMneT>TkeE}UP)tV3aN4o+Au-c5J;^^)F|ZQe$w=`QIZmw^ob;yy>20jR!@ukO z%G)k)H>YPdEWtmX@*8)@Zz7Gfbw3!WqQV-8&Jw5GD4FO=dk2gJ|5{6?;hqWmg|xR7 z&yTN=ctng4l$6WuyzbJs{X%2h?Z`e1B7pxd)5a43D3i{o~ry zs`0LDEZ|W*JQK5YdcOA)Vi)$OE;4mdkVK4^1bXB@BQ#TyVXs~S2?u&V1f{peuE5n! zaele-V85d*tNErq z%us0!5PYe;odR3qpF*7$_J0*a7tn?gDYdliqKM^5i7;328e*gyl3fir*F8(s3bONwWC z>VU8uF3ATG!;Or%6*r5~EyuR;%#xBreWOQ|_XxH21dfl=i%!%Ja<55Bsj40S(6y+h z|NSe6vfwL<5r(EoCw7Qxb3M9zA;gg>_p3Im%mbT9cvX8#+E0Gm^Z| z+Q_n{fwM;pwP=?znxJfaQxO^tE)~OeUwrNtU%C|wrPfKmH(subA3wYjC!L2pymfU7 zAyJ@;SCbBSf&(^l^46%{D4h+k#PVX!gJsY0IyzSmO3od)IliEU5>OQPr@o?k6(&wk z?`I2tTMpr|W$S%`W}CzXww+A!CNGa7L`0lgJ?Mpeae@N#$|SMV$j$s1m8l>ryrbJ( zW~_a(+v6J2j5JR?OBE{`{X~avQ7A?au z)g5M&{mgks=U%hIBRbijjaHQ;2+zP2RduKma$V94-&;6Y^5r`FA_C1&UFO4ggO-@U z>I2;rmMwK1J1or4DL|A|!neJeRuR)E+AIW7ior45>6~3A`TS7{ymhigo!|bA^&d@O zNN@dgJu=GMD zD>TktZ|j&>^e*xw4T>{NPzRL|V{bj%BNzTB;W35xp_A)7GmJ^Y1UIC>2I>?!HTxV2mSU%nM-x3m7jY2g1p;U~J| zQ31y{PXReNaT^*wU~Z5MT}l0N%GysRZ+N_*lJK*4Z?$orXh4=Pl{RujgKlf|iha9N z`zzsXI78mXg#n~;v&x;_Z}si>s~`Ss1z#t7_0k!O!@tvZ6|g$LBOuwY9KT7>un5p<4_p7Eb8=gatfdR1P9`=Ky&p&($@T9yx zG;$l*jr0!;++)eENo}M3y*2wBVUPNc+Ep_^_@J_t@o%-ptL%daK?BJ z5n7J^0xsXZMyReyb5e;cpbBR}KFMDviKWa(dxU}@FZUy7UjWAUch7|y#!qJy$Irk; zJdJ!g;H!;agX2qSnYW{D#eK{~XY>OBvBgouy(i1l(u-=53Nu;!K1?UIT>()umT}1! zZyssVRYANF5^#?8JU(qNer!I$x8w=}{1-{w-mTH5^5py-W>5JUH3^Z|+*6OGv@d!> zftj{cmoU$26v)hh8^Wx=LDb2cqO_KK)r6dIJk-~zC7RKcD*erbSCId<7E(4j?1-?^8ji-r??v)7#T}aVSz#crUhHSs zNzae8bg=!AQ`il`_u0uJ2{%sb_wQxl9G4k`EM>nRRWx-M{v@b&!=v3~@Cg@RD zy6mi71hmXlu=IG`m|^IN%-?nx?56HY8)#aUQV5?RT?(Z<7q6P774Ir~^P#BPku3Mg z9(ck^-|ZtJ_=H7v#x*29j#N{@e`M~FhV?g2;q%`*k=p`Bk_;f-kvfqkJe0oNl#2H) zcpt#Hsz6D(Nf=6>wI-jA5sAab4;U@jJoG)f{iFhvCrm_0i*6ITm(NAslH02lZYp#> z1}K?dYwYgA?(^kpbh5Jso7u7=94|`pRMQo+UT}x=Z-ZKQhRD#n0TmzVmtHacu;K#4 zfVHkk^1&R8-x3{OX)adMX9!U3(KL0B^h1A*7w${ttrN!tS^@J*O-_G+0BF)1>y zReJnr=3$XU1rBMKy%N%!Ja2mh8}Xo|U92GeLg` z<(K;&I~ap{uHh}l;yd@$I@_fc=IFF!QAN_IKbN^j3pG3k@^^mG9c$u^e8mwWvcLnb z<-%MF#%Jc2B-1p6M_&mA6T6leZAVBOa)mS}S$uk)7`_ZhhzgH82ZDHvRd+Jj{AZGU zo-#A}GnXcW?K-)4+yM6q+>x;v_6|h)_GVi$eAHv=(3I)5OXBE7l1EN?r<*-A$x7DT z_2_CZ*GvJfwhUBK(9J3*QkZJ`g7l2eNN%+Pl>q%aEJ}D`@pOqN!6@NDC{0v`mvl|( zwhl9`$q(l&*}p6#7K|l@YrGKF6U$BBZ4fnJ5_OeNE7bOlGgalT_WXA9-t!Z<0R7Cy z-NS=vrgCVatUg#)PujOf6DB;pp*hr<$YJwa66yLp_S(dpa9U1IvStH=?=C0|5$Nm= z`{bK#NW$>|Rk*u#^1ah)tn7}4kSKraDqK=~$4yCWYWdZ$CAx=Wb_urCWdD4?Am3tG zKXgp(h_dR4&&EdRHV)A!{f1>FbDp44h1Pu_J9nz!#abn9q%33WUBs_AANGohw65s# zJ%?XE5nMW6-_#{%$3~aY>K>McZT{LN;taObGs*Cmc`);SGd$Jkflrgb#Qw6@%~ZM} z6F1-Y%cyiZuK{f`$BBH^wt&WZ-#}4c+6FH8aDJUeqn?1Ox_#&D#ZC#Na~280^Z<)- z)_HDs=xU{L!$nFGHHW}Dwyh!-hY+{bm3McbJ^(I5P)dvEP0nvddmL!;y1M03z$M3O@W+qv9BQQT2~luhSV?5L$^w6=O4RO= zCY!L7$nmlYJ1CV@T+HWw;xl6z#h4l$l<-wfc|cR$5v zjN)!ch|{E1ElbuKP1GF`ahX;Qxkav>gu0=tW2dA4>#s2rOx0R(8b5OBCagQ2DHsr1 z(f|Xn?Q#-hJB7vbthUpBmv@i7s>4dVWV~ckI0>~ib{mjVfq$dDXo?_QDl}nLS(qhG zsHy~v7}TsmMaDf~R%0g;4_0c$2hVFB)_0q|l;oS@T)g5ZZ!2Nz_DCp}7JM>SXO)h~ z2CTz?V@-HGq!MYnliDGL^`LjoHj1U0+GQFX4aPL9evSu@j&BjBBxc1zI`}3Rf|v=J zJmH$&ypm*Sz2w9k_dWQU)rDS_lQZ5-x8J(5jcCE4)+PjkLzZL$3Lcv0_=&ON5_ylH z3sZ|N;tIpb2QUd|X;JAaR@KWz8hz-xiydP~z2qkHs*N2S?tgE=kKE%=`BTM?lKLk_z9Y5L^$%2k{y3&T^N9% zr3>#<{>v}kL)=XEJ3Rta)Ul%moup$PV+f98B8Y7^4&{3r>;5+|O2|0HK``=cEy8R1 zJ%V|%p0v06YnjDUVf1eOmsdD5{RMH8RN)hDWN?G87_2n|KBQt4R0Vw#s&7A2ds7)2 zmifxtw?zzKi;`qgNBifku#m?AYx1U^`$~kHzMa=2#{3!k6>C#OjGtE_mWEfC7M!mT zU%yU{r_u`(;!gegM^SIhBMUz+DGM`2+wwN?(G3^3+Pv34q*7*=xeK#VO+_4A3l&Ea zz$hu7!-1(R+Ycn>)vGZkqFIc8;DkP)S4VrOk|1!3gv&@~EC*3BIkn)r@`t=CWm6@) zyapyo)8|1ZG4W-2{PaDYnjh_pt>_~w?0$jG0{#R)wZmA4R?}b9Hc&OA0%qBNT zbLk?`8iHxS6=0I2qAL_lZfcD-sZ+EPvKh@kADavsH5E|@#9tfiK+WQ1TVeNjT?Oc! zV-|KAEP4Sx^?gVnMbs`KC1yy``cJs%afzQgf_E*#JsLUK#lIh0+57}+ZalRMVBQiJ zc{}a>Dp-`1R)Bz@j%If_s?{SW8~>#lgW{dw_GAmTqaA!19->P&RFNg*O2fsC5aF!M z42>GKXE1Z2QFRGwd}aOM?t@%Th8NB3DBsaf5@^*2g9cY&%6zz=g;=Uxvb1!-bnRyy zqyeA$?6`$V44qUjC>x%|JCZa>@$Ep}-_>ikJPERv&KIJ&wijGJR_1Co31f6jsL1@S RLWo{r93CDfAIv<5{~ub%>|Ou> literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-valid-decapsulation b/test/fuzz/qbit_corpora/mlkem_backend_diff/acvp-valid-decapsulation new file mode 100644 index 0000000000000000000000000000000000000000..6fe5560bae189a5f8255ea549ea143776a9c6b64 GIT binary patch literal 6400 zcmeI$1wR~&;{b3@$8?wS+=+8cH68!pW{=4#XZeXfauQc1qVEpkgE`}e zmG3iK=dMr~BuxQSBm3_qeVfk(T>M%t1YIjXvfAdYQ9VP=;HyUs{*d->y#%>c9kAKu zujhFhmnGJk=UQb2Opr7gd_NNrz2G`s5s=(rGY#N#Bpr=u>K)T8$CVD*Z7A&Y<-XoD zPo!;ARI~L?oj(&cn@@KBjmnkSLgl)q{LS^fGcTl5`DYEu%h+w!o1Td`u>Rm8WsaF>yU#=UR@Y56SYJWKA8iFU?=$ z=Zp!qpu(YWNnpCIW|=fyx2lY!<>gj~aZek&bdK>;rCkA)+1loNSIO~{kY-q!pSuSO zrh1|)H$uleWa4={jwYsyb3L|sg{mrpG?SF46ZtE2tIlf+aG&!EbEv1g}sd> zwFrz(PZ-+Ri{v6v1i{f)*&LzfK$XL{pBoxJ9xR~u&@#zU9tDC7cZ*HpWZ*Be=C3oerabj~opXwZN#amsA-&)yER=VS~S|03*WB4e8|V9}02 z^X&+cP&h?AsSG`Ts2h+$78LNwdXv!d`fU8;Pm8BrGp4}A%@s|;kP6}J*NP!5l$$;$ zRI$B>uC1~QC(8>Del*Vz^oKaL-ppym^yFCs!_%?jzqS#QNotV#w>mroAMga`?-fYM z#g4k+f7s(ilrgNCz%1|6Zhk{nxFl%y_G!GltM0gp|t3)0N1J0tbn z7GT||6>YmqT%B+W^Z1lrwwqLbX8$CAs>zF?m%)PWv$`c1xUi1dC%syCKgh6E`_J)S zBODYmRcrW6Nba#c8!;qi_Ym)=yS;w~f^qL=rPQT=$o-YZX ztZjB3QXJb5)u=J(Qj|xkVbHY3_ER>WKPKvs*Aj9t%*O}AOvua!s(K{JU%5FNfz)|| zeI-5vc(GxjaDp3;wi)ga$YuA7%msu-TZSvY|Dd{-e-nuTBqnT|TTj(fRfX_3Ff+QPp_% z6x%7rzU)cs`X7URgiR8Cy%RrVBf=`_aD)$7TMC1a4&ah$rQYV$0^nTj~1r=|(NmIwi5 zMTz3RHhlq(s!eu<)d?^LTv>6O1kTcV2OWrMNML$skl^#Nk|u71W6oD>E#+{FLd8ok%iL|A+}x9#~!`5|vqX#Hw=!0AY3wztSF#e1Bu)Tvi*(Ozg$ zuw465)D1}!akipI&8140CjZry#-z;D$wIj}Ao~4i;Clg;f9{XIw4sTqynx5}e@?Ih zs}Ae$#DaTeDQyP5i3&W@`YhkVeVHEZOTj%D`8at=2W5k5*VaPAEFE?7CiCEJ+7BA^#{HvXRQM&imwHQPC&pvlCL73GxGxi&Yw%y+;eoL(f%IR zOe}kS`+bRx1<~TXd*%>*zhe{Gs7z+`!?mlplded!;YUd?3^^jBNFKd2cl)j}0Z4z8 zDHu7Iczw(32FQis`!?Xi-**ex43q_0-(#6^mi=?D0eYko9W-~N{>+g}) zGwvCW-we6myM2g|&@mUv;To$|OT>*=7!ftm`K`|+iXa=1lP^;YfH#t<_b2(t`$3!8 zCb;^A15DK_W8zE>((bIg>DPE1rMuOBRrR{-F3U!kzT$mkHPyQpvG~qzarL%xv^4^Z z?c&^Bv~`K#C>edi@MSr}=feVr2KS6T(bdhh5*)-BSeawYas=jYXs&oZ zvionVYkre%I9i*&lp`ZTR|A1j_L+a>{0~-sfH7j6h(+%@?aYx$;Wk@qsB$ELlgEOt z?Cuz%@FXwI&%UXdsOK*?v2oz+UTvcasoF=ww1imtQ@4 z!V12zj)goZg(A>o*bT>}#ZR_C5M5o#n<7wB>G%?Tw#(G2XaA5;UQESmyh&z)1Dvu{ z@5@r3=oTmCFODs!*XH!lhvqsHzB@;tF=V`pqLCWvg_OqKL#eJTT^pPt`A>k?0p2;= zQz-WyQF}C4p8xL!%LzJt;C%DD3zo>^vj0mMbAadZUr zDhR)}n}*Jj(MxC!=Es-{yey_!m0W2jF#l}^mCpFyvZjb@Sy%e%3G5Zil-$Pq*vR;= z8+44sF1aky0u}f4D=`Utsej`mJ_eXQ$$b|-0I5ci#!R@yY);H>DaN45*|u9M)jWe7 zy>_f3Or>F_#$|qEHloGo2XBJU1SO)MjwGTS+=1-v;i9gw!f@Wzi@&gAmr>8RYAewG zf8+f&*ifHgJfh95r`>){pQvDe*h@CE9MA#lJQ98sjOShB@By^BupKHgvf~oZMwiw& zEzaqz&gpHCJ*2U2XlcZbb4MPt7H9QRXaYC*lngb$0{UP@Ec^Q7$x{7i#1%^x>rgEf z2T@{e7)uY}`ID0Dw+Yq!bK`5N@iEX&&eW$E!k@dh>oR$c^Gj>oLL|zSw0~Wgp(<;; zwaghUlZ<1})Rq?bWhy|pFr^FSQ4qzb{fqFm=D4(I*=JZ6KJ5bIM=7LDZx6B5c-_d9LSoIgT2P~ z39|A2Pk!^z-1lB~`Fs@0LyJ>1&69IufkPSS8!#A1x?xYMMo{k=Hj>a4d5JNmoPF25 z4Z7u?b;BfG^fM9u>xm%pQ%EEFJpU4qWo3LQFy$eGE{Je%XGW-LB465tT&Tkymn&=J z>hN}6urlPe(;DsHDteFEB-=TQbXRZZERY<@ThI8g^L7X5rm|>gJj#{gqT3%&Ypn|e zb%c9gmw4bqoQVO^3|nuXZ~1?Ssa`#aR+_+Jt}T0}%M^Peag3*+=td&OWvXM6@|RH- z+{N3OukTn)W*Cf}#)(tNLIWz)HSn5wQyTv@E%U;2iN?-?6^8i3e00=(=DHn>zbm@P zOS{a#g+_g)80YdPM&|O-LC^qs$n5zs`762??X_Zqw`K1t|U>21mDvX!OrJx z>o^1PlN^l$k^C}Tp}t)lV{x@B6(sMU#^Z;B*8K0zr?Nn}{&?-fL2f_0flVouqBVU# zcn3cq8Sn3GKZg}?Y@j&yx}$`H2FPh2glv;Tu(whQxps=Qc2dK)dMWgk4vZo6fhe6_d-4ViylJzJRU+B;rg!G#&Z|Biw2@MVj(7mUF&A%9 zt3WzJIh+o-hJD)saeu0e?-QVB3lVGq4+pd`ojTiUNJVlt>8zkgZHyO zLXFqH0XGR@HLQdjLPz6S6r;7cm;x>(^HEBzmy*DqnBnP^8VgN{3ptI5#&Nfv2sb%3 zcXoR6!t9!EZqul|hF6^_ylnMw%&JT<#Gm@s$qF~`b|{IJo95a^oM6hFJk+S!wCZd| z+q#&*TitSLhi)2*DvJOBtW<1=h>ceUYN35qC`|p)Qej>xIj=$z8uBEjDmQ^XPF6*v%EfzI-vvs4j(nlWaN5 zx`@j;1&C;Bkwg%V{xRb%XR_p@^3AoWsiZ#6IA!hm z^#OEo2Zdc&QXvhby%iR7%`+3{jk@)UDNi-AF-!`fKN_++*G>N*5&7bRa?)B6`U|ud zW|!$P#nZX?lHu)NPc4e+Y(ZduK)h<;>jvAC&i>fN(F&P$8#>U8J(PO92s9^Sw ziw23OIU2$P_CLpv$-Hpx#<2eycTI$<4zaJBh{gp@j4|+kJ7D|$d};jzJ|F4vc0K1$ zIV$JU8J=pR7OpMjrCiB>fArrU{r5-z{n3Aa^xq%- ze}7b+qv;e=U$$bJxe;gYqx!73Pf&$fsL*Kf0UFl)o1ao9 TaT7I*^+w-KGY6D%?xy|^oqFqc literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/cctv-base-key b/test/fuzz/qbit_corpora/mlkem_backend_diff/cctv-base-key new file mode 100644 index 0000000000000000000000000000000000000000..f4eb8e847187176031bccc800c98f954df4e8137 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoARom;=uM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3328 b/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3328 new file mode 100644 index 0000000000000000000000000000000000000000..21a0a6fa803542c309e985ea41a5bc211ff77070 GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR0MFX7x literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3329 b/test/fuzz/qbit_corpora/mlkem_backend_diff/coefficient-3329 new file mode 100644 index 0000000000000000000000000000000000000000..037364c402af2300a1fc628a5b5bf506490e915b GIT binary patch literal 1664 zcmV-`27mcsp4%8zlFtlDhQk0odbjj3ZmZ}eVxQa@R+G>TN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355y|h|N2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR6Wdoi7 literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/mlkem_backend_diff/honest-ciphertext-bit-flip b/test/fuzz/qbit_corpora/mlkem_backend_diff/honest-ciphertext-bit-flip new file mode 100644 index 0000000000000000000000000000000000000000..e90fa0ac39fb8f88ac32e96c5e5c7600b649e4b6 GIT binary patch literal 6403 zcmYeCcUvNK(s>@ww!;jzRolN9=B<9Anl%58WZ2{jyk6}`811Ted^O5n^HME&!Ck5F zDHr*?JB~8h*X;aeT(I_)dP?EC*BYsd?#o0@yDZ?_b)3bqZubwTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaTN{7S&K6|+JGHL+AzuIngeqTd`@meUYTipK;&e!KWJaG$~)UjtQJ^cJHW`V&^6GdL0&9F5Y|VwqnJWO!{c=zU0dCr$xYkDGza)eorZG`sO%$dbq^ z)M_E74)0@HsGO+DxX+YIS98h%NQ(;7TB)mBtK`7VW2p-!MW{W?>N_jbY1uv=vX+vyf6~zgPo5EF- z&ZD-J*eb?V7AA{Wpzx*2`v*MvLkXKzk8Dwz3zqyqliTt(EF*Yw77cu-wkKd}(a08) zAShcJJ>Izucm^0dE6nC1jJ2U=kV;{J$B6_M60d`@z%wSQ_{3<&qWUVr*cdIMlpTLG zCSKJ_3dXqj8OfKXY$nu<76@vccd}i8a9yIm8G{A7F%_i>##JRSM8qZ@8y#|oYOYza z<~N7GDzJAvmC$f=A*+$KA#Ia&2DIa||=hlT)k&g^dbUuYAum0F&+oFqbxYo`;eFgfqWYkQH>pL9(ge9LlnlRwU(*1QQ2K z7!dhay&x)bh*i2A5;_e#$cRU8;oT< zTMH9gnd~IN8=l!ZxE%M8-i=SW^ixQx83HQDT7Py7;j7Tx2xg@7gLGR zv9-y@bmsca*)dr0b|~Sj5tp+G0<$M`X0HbVpjToN&)IN$M?y()zq@q-9J(>GrHr6r zqSXR?i!#1q=uLjAsajt|`!0V!~xW(Z)E4x7Nu7eA3J4`KmX86mlyaEU0|q zS)I;OPNu}TJkudF!)>@0E<7}7AKH$h$I0jnhto=PkyEs%V!#pOHe;W`UNi(4DiKQ?u0xoh!{sK5Gs+)di+}bmWQSn zWoKmw-Vjir{I|QPEeAjbI=}(zGj4(VT42Pd6gsN<`8r}Dj&=*3-q^%SYfb4neVk`| zt=W_>G;dpgBCL!mRK++}B1_uf6k#@FYRn5r6}^y)YsDqN@MkguOMybcWRTPu{Dp@; z>sIo>0`U@Svj<;Wv22-SE`arWP4GvWR!&gIh%-o8Hl{o{(aVW4$ps1ude=%+@}h@` zy5NGzM%tc@b$b37fVI_hcNb}a15<#Qv3EFaHrKzoY@w#MjcU5QT17GHGs^`sdAJ;e z#?)J}(Mq8rSo}bJ=BcIKW~t$8Yu355z4ZS|2G?UnTTP+>0X}FPL+X5#KCy=F@|I-9 K-9q~hBoAR`T?DiM literal 0 HcmV?d00001 diff --git a/test/fuzz/qbit_corpora/overlay.py b/test/fuzz/qbit_corpora/overlay.py index ce00d7d3df..0533e6507f 100755 --- a/test/fuzz/qbit_corpora/overlay.py +++ b/test/fuzz/qbit_corpora/overlay.py @@ -29,6 +29,8 @@ "asert_edge_cases", "asert_math", "auxpow", + "mlkem", + "mlkem_backend_diff", "p2mr_script", "pqc", ) diff --git a/test/fuzz/test_qbit_fuzz_runner.py b/test/fuzz/test_qbit_fuzz_runner.py index d4066f98ea..23a3d8a0da 100755 --- a/test/fuzz/test_qbit_fuzz_runner.py +++ b/test/fuzz/test_qbit_fuzz_runner.py @@ -31,7 +31,7 @@ SRC_DIR = Path(__file__).resolve().parents[2] RUNNER = SRC_DIR / "test" / "fuzz" / "test_runner.py" CORPORA_DIR = SRC_DIR / "test" / "fuzz" / "qbit_corpora" -REQUIRED = ("asert_chain_transition", "asert_edge_cases", "asert_math", "auxpow", "p2mr_script", "pqc") +REQUIRED = ("asert_chain_transition", "asert_edge_cases", "asert_math", "auxpow", "mlkem", "mlkem_backend_diff", "p2mr_script", "pqc") INTEGRATION = {} STUB_SOURCE = r''' diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 12fb595ad4..d04b655c9d 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -24,6 +24,8 @@ "asert_edge_cases", "asert_math", "auxpow", + "mlkem", + "mlkem_backend_diff", "p2mr_script", "pqc", ) diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 77aed1aa8a..84facc2482 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -12,6 +12,7 @@ unsigned-integer-overflow:*/include/c++/ unsigned-integer-overflow:FuzzedDataProvider::ConsumeIntegralInRange unsigned-integer-overflow:leveldb/ unsigned-integer-overflow:minisketch/ +unsigned-integer-overflow:mlkem-native/ unsigned-integer-overflow:secp256k1/ unsigned-integer-overflow:test/fuzz/crypto_diff_fuzz_chacha20.cpp implicit-integer-sign-change:*/include/boost/ @@ -20,18 +21,22 @@ implicit-integer-sign-change:*/new_allocator.h implicit-integer-sign-change:*/qarraydata.h implicit-integer-sign-change:crc32c/ implicit-integer-sign-change:minisketch/ +implicit-integer-sign-change:mlkem-native/ implicit-integer-sign-change:secp256k1/ implicit-signed-integer-truncation:*/include/c++/ implicit-signed-integer-truncation:leveldb/ +implicit-signed-integer-truncation:mlkem-native/ implicit-signed-integer-truncation:secp256k1/ implicit-signed-integer-truncation,implicit-integer-sign-change:secp256k1_modinv64_posdivsteps_62_var implicit-unsigned-integer-truncation:*/include/c++/ implicit-unsigned-integer-truncation:leveldb/ +implicit-unsigned-integer-truncation:mlkem-native/ implicit-unsigned-integer-truncation:secp256k1/ implicit-unsigned-integer-truncation:test/fuzz/crypto_diff_fuzz_chacha20.cpp shift-base:*/include/c++/ shift-base:leveldb/ shift-base:minisketch/ +shift-base:mlkem-native/ shift-base:secp256k1/ shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp From 7b36ae48653a1b259a121ee31a3025077e9aa2ca Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 13:26:18 -0400 Subject: [PATCH 06/20] bench: measure ML-KEM operations by active backend Benchmark ML-KEM-1024 key generation, encapsulation and decapsulation twice each: on the detected backend and with portable C forced. Like the per-implementation SHA256 benchmarks, each is named after the backend that ran, so on an AVX2 host bench_qbit -filter='MLKEM.*' lists both, and the portable guard restores auto-detection afterwards. Part of #184 (part 3, #190, D5). Co-Authored-By: Claude Opus 5.5 --- src/bench/CMakeLists.txt | 1 + src/bench/mlkem.cpp | 107 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 src/bench/mlkem.cpp diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index afa66f354a..52d45676be 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -38,6 +38,7 @@ add_executable(bench_bitcoin mempool_eviction.cpp mempool_stress.cpp merkle_root.cpp + mlkem.cpp obfuscation.cpp parse_hex.cpp peer_eviction.cpp diff --git a/src/bench/mlkem.cpp b/src/bench/mlkem.cpp new file mode 100644 index 0000000000..0be1a0d024 --- /dev/null +++ b/src/bench/mlkem.cpp @@ -0,0 +1,107 @@ +// Copyright (c) 2026-present The qbit core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit. + +#include +#include +#include + +#include +#include +#include +#include + +using namespace mlkem; + +namespace { + +enum class Backend { DETECTED, PORTABLE }; + +/** + * Run `body` on the detected backend, or with portable C forced, and name the + * benchmark after the backend that runs. Auto-detection is restored afterwards. + */ +template +void WithBackend(benchmark::Bench& bench, const char* operation, Backend backend, Body body) +{ + std::optional portable; + if (backend == Backend::PORTABLE) portable.emplace(); + const BackendNames names{GetBackendNames()}; + bench.name(strprintf("%s using the '%s/%s' ML-KEM-1024 backend", operation, names.arith, names.keccak)); + body(); +} + +std::array Seed() +{ + std::array seed; + for (size_t i{0}; i < seed.size(); ++i) seed[i] = uint8_t(i * 7 + 1); + return seed; +} + +void KeyGenBench(benchmark::Bench& bench, Backend backend) +{ + WithBackend(bench, "MLKEMKeyGen", backend, [&] { + auto seed{Seed()}; + PublicKey ek; + DecapsulationKey dk; + bench.batch(1).unit("keygen").run([&] { + ++seed[0]; + const Error error{KeyGen(seed, ek, dk)}; + assert(error == Error::NONE); + }); + }); +} + +void EncapsBench(benchmark::Bench& bench, Backend backend) +{ + WithBackend(bench, "MLKEMEncaps", backend, [&] { + PublicKey ek; + DecapsulationKey dk; + const Error keygen{KeyGen(Seed(), ek, dk)}; + assert(keygen == Error::NONE); + std::array coins{}; + Ciphertext ct; + SharedSecret ss; + bench.batch(1).unit("encaps").run([&] { + ++coins[0]; + const Error error{Encaps(ek, coins, ct, ss)}; + assert(error == Error::NONE); + }); + }); +} + +void DecapsBench(benchmark::Bench& bench, Backend backend) +{ + WithBackend(bench, "MLKEMDecaps", backend, [&] { + PublicKey ek; + DecapsulationKey dk; + const Error keygen{KeyGen(Seed(), ek, dk)}; + assert(keygen == Error::NONE); + const std::array coins{}; + Ciphertext ct; + SharedSecret sent; + const Error encaps{Encaps(ek, coins, ct, sent)}; + assert(encaps == Error::NONE); + SharedSecret received; + bench.batch(1).unit("decaps").run([&] { + const Error error{Decaps(dk, ct, received)}; + assert(error == Error::NONE); + }); + }); +} + +} // namespace + +static void MLKEM_KEYGEN_DETECTED(benchmark::Bench& bench) { KeyGenBench(bench, Backend::DETECTED); } +static void MLKEM_KEYGEN_PORTABLE(benchmark::Bench& bench) { KeyGenBench(bench, Backend::PORTABLE); } +static void MLKEM_ENCAPS_DETECTED(benchmark::Bench& bench) { EncapsBench(bench, Backend::DETECTED); } +static void MLKEM_ENCAPS_PORTABLE(benchmark::Bench& bench) { EncapsBench(bench, Backend::PORTABLE); } +static void MLKEM_DECAPS_DETECTED(benchmark::Bench& bench) { DecapsBench(bench, Backend::DETECTED); } +static void MLKEM_DECAPS_PORTABLE(benchmark::Bench& bench) { DecapsBench(bench, Backend::PORTABLE); } + +BENCHMARK(MLKEM_KEYGEN_DETECTED, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_KEYGEN_PORTABLE, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_ENCAPS_DETECTED, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_ENCAPS_PORTABLE, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_DECAPS_DETECTED, benchmark::PriorityLevel::HIGH); +BENCHMARK(MLKEM_DECAPS_PORTABLE, benchmark::PriorityLevel::HIGH); From 635265d173bb8fb4c123a4e6ae8ab6050f9cd877 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 13:33:18 -0400 Subject: [PATCH 07/20] ci: add s390x and macOS arm64 PQ unit jobs Add two jobs to the PQ workflow, selectable by name like aarch64-unit: - s390x-unit cross-compiles only test_bitcoin for big-endian s390x on the x64 runner, with Ubuntu's g++-s390x-linux-gnu and the depends tree, and runs the suites through ctest with qemu-s390x as the crosscompiling emulator, so only the tests run emulated. It stays out of the merge gate. Its timeout is 180 minutes until runs are measured. - macos-arm64-unit runs natively on a GitHub-hosted macos-14 runner without Docker, asserts that uname -m is arm64, and shares the unit job's source checks and evidence report through YAML anchors. It is the job that runs native AArch64 ML-KEM, and configures with WITH_MLKEM_NATIVE=ON so that losing native code fails it. All three jobs now expect mlkem_tests besides bip324_tests and net_tests (CTEST_REGEX ^(mlkem_tests|bip324_tests|net_tests)$), and report the suites through ctest_evidence.py. The aarch64 job's description says that ML-KEM runs portable C on AArch64 Linux. The PQ CI contract test covers selecting each job, the s390x cross and emulator settings, the macOS architecture check and report, and that every expression uses a context GitHub makes available there (no runner in job env, no matrix in a job without one), which GitHub enforces when loading the file. Part of #184 (part 3, #190, C2 and the CI part of D3). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-pq.yml | 92 ++++++++++-- ci/checks/test_ci_pq_contract.py | 163 ++++++++++++++++++++-- ci/test/00_setup_env_mac_native_pq.sh | 26 ++++ ci/test/00_setup_env_native_aarch64_pq.sh | 2 +- ci/test/00_setup_env_s390x_unit.sh | 31 ++++ 5 files changed, 287 insertions(+), 27 deletions(-) create mode 100755 ci/test/00_setup_env_mac_native_pq.sh create mode 100755 ci/test/00_setup_env_s390x_unit.sh diff --git a/.github/workflows/ci-pq.yml b/.github/workflows/ci-pq.yml index c252804291..e16273bda1 100644 --- a/.github/workflows/ci-pq.yml +++ b/.github/workflows/ci-pq.yml @@ -12,6 +12,7 @@ # That caller lands on main after this part merges. # The workflow_dispatch trigger below stays for when this file is on main itself; # until then GitHub rejects gh workflow run ci-pq.yml. +# Jobs: aarch64-unit, s390x-unit (big-endian, emulated) and macos-arm64-unit. # The source is resolved once to a commit, every job checks out and verifies # that commit, and the summaries record it. @@ -27,7 +28,7 @@ on: type: string default: '' jobs: - description: 'Comma-separated jobs to run (aarch64-unit); empty runs all' + description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit); empty runs all' required: false type: string default: '' @@ -39,7 +40,7 @@ on: type: string default: '' jobs: - description: 'Comma-separated jobs to run (aarch64-unit); empty runs all' + description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit); empty runs all' required: false type: string default: '' @@ -59,6 +60,7 @@ jobs: matrix: ${{ steps.select.outputs.matrix }} matrix_count: ${{ steps.select.outputs.matrix_count }} selected: ${{ steps.select.outputs.selected }} + run_macos_arm64_unit: ${{ steps.select.outputs.run_macos_arm64_unit }} steps: - name: Resolve source id: resolve @@ -167,15 +169,23 @@ jobs: [ { "job": "aarch64-unit", - "name": "AArch64 unit tests (pre-library phase)", + "name": "AArch64 unit tests", "runs-on": "ubuntu-24.04-arm", "timeout-minutes": 120, "file-env": "./ci/test/00_setup_env_native_aarch64_pq.sh", - "phase": "Pre-library phase: bip324_tests and net_tests on AArch64. Part 3 (#190) adds mlkem_tests with the ML-KEM library." + "phase": "mlkem_tests, bip324_tests and net_tests on AArch64 Linux. ML-KEM runs portable C here: its AArch64 assembly has no BTI landing pads." + }, + { + "job": "s390x-unit", + "name": "s390x big-endian unit tests (emulated)", + "runs-on": "ubuntu-24.04", + "timeout-minutes": 180, + "file-env": "./ci/test/00_setup_env_s390x_unit.sh", + "phase": "mlkem_tests, bip324_tests and net_tests cross-built for big-endian s390x and run under qemu-s390x. The timeout stays 180 minutes until runs are measured, then becomes twice the measured time." } ] # Selectable jobs that are not matrix entries; each gets a run_ output. - NON_MATRIX_JOBS: '' + NON_MATRIX_JOBS: 'macos-arm64-unit' run: | # Turn the comma-separated jobs input into the matrix: empty selects # every job; an empty, unknown or repeated name fails the run. @@ -233,6 +243,8 @@ jobs: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 DANGER_CI_ON_HOST_FOLDERS: 1 FILE_ENV: ${{ matrix.file-env }} + # Read by the steps macos-arm64-unit shares, so it is set per job. + MATRIX_NAME: ${{ matrix.name }} TEST_RUNNER_FAILFAST: 'false' MAKEJOBS: '-j4' TEST_RUNNER_JOBS: '4' @@ -245,7 +257,8 @@ jobs: CI_BUILDKIT_DNS_NAMESERVERS: '1.1.1.1,8.8.8.8' CI_BUILDKIT_DNS_SEARCH_DOMAINS: '' steps: - - name: Require resolved source + - &REQUIRE_SOURCE + name: Require resolved source shell: bash env: RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} @@ -258,20 +271,21 @@ jobs: || { echo "::error::resolved source '$RESOLVED_SHA' is not a full commit hash" >&2; exit 1; } [ -n "$REQUESTED_REF" ] || { echo "::error::requested ref is empty" >&2; exit 1; } - - name: Checkout + - &CHECKOUT_SOURCE + name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 with: ref: ${{ needs.resolve-source.outputs.resolved_sha }} fetch-depth: 0 - - name: Verify checked-out source + - &VERIFY_SOURCE + name: Verify checked-out source shell: bash env: SOURCE_MODE: ${{ needs.resolve-source.outputs.source_mode }} REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} WORKFLOW_SHA: ${{ github.workflow_sha }} - MATRIX_NAME: ${{ matrix.name }} run: | set -euo pipefail actual_head="$(git rev-parse --verify 'HEAD^{commit}')" @@ -310,9 +324,8 @@ jobs: env: RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} - MATRIX_NAME: ${{ matrix.name }} PHASE: ${{ matrix.phase }} - run: | + run: &REPORT_EVIDENCE | set -euo pipefail # ci/test/03_test_script.sh writes this after ctest_evidence.py has # checked that every expected suite was selected, executed and passed. @@ -333,3 +346,60 @@ jobs: - name: Save caches uses: ./.github/actions/save-caches + + # Apple Silicon runs natively on the host, without Docker, so it is a job of + # its own rather than a matrix entry of unit. It shares unit's source checks + # and evidence report. + macos-arm64-unit: + name: macOS arm64 unit tests + needs: resolve-source + if: ${{ needs.resolve-source.outputs.run_macos_arm64_unit == 'true' }} + runs-on: macos-14 + timeout-minutes: 120 + env: + DANGER_RUN_CI_ON_HOST: 1 + FILE_ENV: ./ci/test/00_setup_env_mac_native_pq.sh + MATRIX_NAME: macOS arm64 unit tests + BASE_ROOT_DIR: ${{ github.workspace }} + TEST_RUNNER_FAILFAST: 'false' + MAKEJOBS: '-j3' + TEST_RUNNER_JOBS: '3' + CTEST_JOBS: '3' + steps: + - *REQUIRE_SOURCE + - *CHECKOUT_SOURCE + - *VERIFY_SOURCE + + - name: Require Apple Silicon + shell: bash + run: | + set -euo pipefail + machine="$(uname -m)" + if [ "$machine" != "arm64" ]; then + echo "::error::macos-arm64-unit must run on arm64, but uname -m reports '$machine'" >&2 + exit 1 + fi + echo "uname -m: $machine" + + # The runner context is not available in job-level env, so the build + # directory, which the report step reads, is set here. + - name: Set build directory + shell: bash + run: echo "BASE_BUILD_DIR=${RUNNER_TEMP}/build" >> "$GITHUB_ENV" + + - name: Install Homebrew packages + env: + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + run: brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent + + - name: CI script + run: ./ci/test_run_all.sh + + - name: Report unit-test evidence + if: ${{ always() }} + shell: bash + env: + RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} + REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} + PHASE: mlkem_tests, bip324_tests and net_tests natively on Apple Silicon, where ML-KEM runs its native AArch64 code. + run: *REPORT_EVIDENCE diff --git a/ci/checks/test_ci_pq_contract.py b/ci/checks/test_ci_pq_contract.py index 3228aac485..bebd315bc7 100755 --- a/ci/checks/test_ci_pq_contract.py +++ b/ci/checks/test_ci_pq_contract.py @@ -2,7 +2,7 @@ # Copyright (c) 2026-present The qbit core developers # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. -"""Executable contract for the PQ CI entry points (#184 part 2, work item C1). +"""Executable contract for the PQ CI entry points (#184 part 2, work item C1; part 3, C2). ``ci-pq.yml`` and ``ci-nightly-heavy.yml`` are reusable and job-selectable, and the PQ unit job proves which suites ran. These tests read the real workflow @@ -64,7 +64,12 @@ REPORT_STEP = "Report unit-test evidence" PQ_UNIT_JOB = "unit" AARCH64_JOB = "aarch64-unit" -AARCH64_SUITES = ["bip324_tests", "net_tests"] +S390X_JOB = "s390x-unit" +MACOS_JOB = "macos-arm64-unit" +PQ_MATRIX_JOBS = [AARCH64_JOB, S390X_JOB] +ALL_PQ_JOBS = {PQ_UNIT_JOB: PQ_MATRIX_JOBS, MACOS_JOB: []} +PQ_SUITES = ["mlkem_tests", "bip324_tests", "net_tests"] +AARCH64_SUITES = PQ_SUITES NIGHTLY_MATRIX_JOBS = ["arm32-unit-1", "arm32-unit-2", "previous-releases", "fuzz"] NIGHTLY_ALL_JOBS: dict[str, list[str]] = {"nightly-matrix": NIGHTLY_MATRIX_JOBS, "scanner-readiness": []} NIGHTLY_VARS_OFF = {"CI_NIGHTLY_HEAVY_RUN_TEST_MATRIX": "false", "CI_NIGHTLY_HEAVY_RUN_SCANNERS": "false"} @@ -74,7 +79,7 @@ MATRIX_EXPRESSION = "${{ fromJSON(needs." + RESOLVER_JOB + ".outputs.matrix) }}" ANNOTATED_TAG = "v9.9.9-annotated" AMBIGUOUS_NAME = "dup" -SUITE_INVENTORY = ["addrman_tests", "bip324_tests", "bip324_tests_extra", "net_tests", "netbase_tests", "util_tests"] +SUITE_INVENTORY = ["addrman_tests", "bip324_tests", "bip324_tests_extra", "mlkem_tests", "net_tests", "netbase_tests", "util_tests"] # --------------------------------------------------------------------------- @@ -455,7 +460,10 @@ def test_selection_scripts_are_identical(self) -> None: def test_named_dispatch_runs_only_selected_jobs(self) -> None: cases: list[tuple[str, dict[str, Any], dict[str, list[str]]]] = [ (PQ, {"jobs": AARCH64_JOB}, {PQ_UNIT_JOB: [AARCH64_JOB]}), - (PQ, {"jobs": ""}, {PQ_UNIT_JOB: [AARCH64_JOB]}), + (PQ, {"jobs": S390X_JOB}, {PQ_UNIT_JOB: [S390X_JOB]}), + (PQ, {"jobs": MACOS_JOB}, {MACOS_JOB: []}), + (PQ, {"jobs": f" {MACOS_JOB} , {AARCH64_JOB} "}, {PQ_UNIT_JOB: [AARCH64_JOB], MACOS_JOB: []}), + (PQ, {"jobs": ""}, ALL_PQ_JOBS), (NIGHTLY, {"run_test_matrix": True, "run_scanners": False, "jobs": "previous-releases"}, {"nightly-matrix": ["previous-releases"]}), (NIGHTLY, {"jobs": ""}, {"nightly-matrix": NIGHTLY_MATRIX_JOBS, "scanner-readiness": []}), @@ -488,7 +496,7 @@ def test_scheduled_call_from_main_runs_everything_on_the_maintained_branch(self) outputs = context["needs"][RESOLVER_JOB]["outputs"] self.assertEqual(outputs["resolved_sha"], self.fixture.maintained_sha) self.assertNotEqual(outputs["resolved_sha"], context["github"]["sha"]) - self.assertEqual(running_jobs(self.workflows[PQ], context), {PQ_UNIT_JOB: [AARCH64_JOB]}) + self.assertEqual(running_jobs(self.workflows[PQ], context), ALL_PQ_JOBS) # -- called and direct nightly runs --------------------------------------------- @@ -729,7 +737,8 @@ def group(context: dict[str, Any]) -> str: def test_rejects_unknown_repeated_and_empty_job_names(self) -> None: bad = { - PQ: ["aarch64", "AARCH64-UNIT", f"{AARCH64_JOB},{AARCH64_JOB}", f"{AARCH64_JOB},", ",", "s390x-unit"], + PQ: ["aarch64", "AARCH64-UNIT", f"{AARCH64_JOB},{AARCH64_JOB}", f"{AARCH64_JOB},", ",", "s390x", "macos-arm64", + f"{MACOS_JOB},{MACOS_JOB}"], NIGHTLY: ["previous_releases", "previous-releases,previous-releases", "previous-releases,,fuzz", "unit"], } for key, values in bad.items(): @@ -784,7 +793,11 @@ def test_every_pq_job_checks_out_the_resolved_commit(self) -> None: workflow = self.workflows[PQ] context = self.resolve(PQ, self.dispatch(PQ, source_ref=svc.MAINTAINED_BRANCH, jobs=AARCH64_JOB)) refs = svc.check_source_wiring(workflow, context) - self.assertEqual(refs, {PQ_UNIT_JOB: fixture.maintained_sha}) + self.assertEqual(refs, {PQ_UNIT_JOB: fixture.maintained_sha, MACOS_JOB: fixture.maintained_sha}) + # The macOS job shares the unit job's guard, checkout and verification steps. + for step_name in (svc.GUARD_STEP, svc.CHECKOUT_STEP, svc.VERIFY_STEP): + self.assertEqual(svc.find_step(workflow["jobs"][MACOS_JOB], step_name), + svc.find_step(workflow["jobs"][PQ_UNIT_JOB], step_name), step_name) mutated = copy.deepcopy(workflow) checkout = svc.find_step(mutated["jobs"][PQ_UNIT_JOB], svc.CHECKOUT_STEP) @@ -812,7 +825,7 @@ def test_aarch64_job_runs_on_a_public_arm_runner(self) -> None: workflow = self.workflows[PQ] (entry,) = [entry for entry in catalog(workflow) if entry["job"] == AARCH64_JOB] self.assertEqual(entry["runs-on"], "ubuntu-24.04-arm") - self.assertIn("pre-library", entry["name"]) + self.assertIn("mlkem_tests", entry["phase"]) env_file = entry["file-env"] self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) @@ -834,7 +847,7 @@ def test_aarch64_job_runs_on_a_public_arm_runner(self) -> None: self.assertEqual(effective["CI_IMAGE_NAME_TAG"], "docker.io/library/ubuntu:24.04") self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") regex = re.compile(effective["CTEST_REGEX"]) - self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], AARCH64_SUITES) + self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], sorted(AARCH64_SUITES)) # 02_run_container.py passes only variables exported by some ci/test/00_setup_env*.sh into the container. exported = set() @@ -843,6 +856,126 @@ def test_aarch64_job_runs_on_a_public_arm_runner(self) -> None: for name in ("CI_BUILD_TARGET", "CTEST_REGEX", "CTEST_EXPECTED_SUITES", "CTEST_INCLUDE_RANGE"): self.assertIn(name, exported) + def test_s390x_job_cross_compiles_and_emulates_only_the_tests(self) -> None: + workflow = self.workflows[PQ] + (entry,) = [entry for entry in catalog(workflow) if entry["job"] == S390X_JOB] + self.assertEqual(entry["runs-on"], "ubuntu-24.04", "built on an x64 runner, not in an emulated container") + self.assertEqual(entry["timeout-minutes"], 180) + env_file = entry["file-env"] + self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) + + context = dict(self.resolve(PQ, self.dispatch(PQ, jobs=S390X_JOB)), matrix=entry) + env = svc.job_env(workflow, workflow["jobs"][PQ_UNIT_JOB], context) + self.assertEqual(env["FILE_ENV"], env_file) + names = ["HOST", "PACKAGES", "DEP_OPTS", "NO_DEPENDS", "CI_IMAGE_PLATFORM", "CI_BUILD_TARGET", "CTEST_REGEX", + "CTEST_EXPECTED_SUITES", "CTEST_INCLUDE_RANGE", "RUN_FUNCTIONAL_TESTS", "BITCOIN_CONFIG"] + effective = source_env(env_file, names, {key: env[key] for key in ("CI_IMAGE_REGISTRY_PREFIX", "CI_ENFORCE_INTERNAL_REGISTRY")}) + self.assertEqual(effective["HOST"], "s390x-linux-gnu") + self.assertEqual(effective["NO_DEPENDS"], "", "the depends tree is built for s390x") + self.assertEqual(effective["CI_IMAGE_PLATFORM"], "", "only the tests run emulated") + self.assertIn("g++-s390x-linux-gnu", effective["PACKAGES"].split()) + self.assertIn("qemu-user", effective["PACKAGES"].split()) + self.assertEqual(effective["CI_BUILD_TARGET"], "test_bitcoin") + self.assertEqual(effective["CTEST_EXPECTED_SUITES"].split(), PQ_SUITES) + self.assertEqual(effective["CTEST_INCLUDE_RANGE"], "") + self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") + regex = re.compile(effective["CTEST_REGEX"]) + self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], sorted(PQ_SUITES)) + # ctest runs each test through the emulator, so test_qbit is never started directly. + self.assertIn("-DCMAKE_CROSSCOMPILING_EMULATOR='qemu-s390x;-L;/usr/s390x-linux-gnu'", effective["BITCOIN_CONFIG"]) + + def test_macos_job_runs_natively_on_apple_silicon(self) -> None: + fixture = self.fixture + workflow = self.workflows[PQ] + job = workflow["jobs"][MACOS_JOB] + self.assertNotIn(MACOS_JOB, [entry["job"] for entry in catalog(workflow)]) + self.assertEqual(job["runs-on"], "macos-14") + context = self.resolve(PQ, self.dispatch(PQ, jobs=MACOS_JOB)) + env = svc.job_env(workflow, job, context) + self.assertEqual(env["DANGER_RUN_CI_ON_HOST"], "1", "no Docker on the macOS host") + env_file = env["FILE_ENV"] + self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) + self.assertEqual(env["MATRIX_NAME"], job["name"]) + names = ["CI_OS_NAME", "NO_DEPENDS", "CI_BUILD_TARGET", "CTEST_REGEX", "CTEST_EXPECTED_SUITES", "RUN_FUNCTIONAL_TESTS", + "BITCOIN_CONFIG"] + effective = source_env(env_file, names) + # Losing the native AArch64 backend must fail the configure step, not fall back quietly. + self.assertIn("-DWITH_MLKEM_NATIVE=ON", effective["BITCOIN_CONFIG"].split()) + self.assertEqual(effective["CI_OS_NAME"], "macos") + self.assertEqual(effective["NO_DEPENDS"], "1") + self.assertEqual(effective["CI_BUILD_TARGET"], "test_bitcoin") + self.assertEqual(effective["CTEST_EXPECTED_SUITES"].split(), PQ_SUITES) + self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") + regex = re.compile(effective["CTEST_REGEX"]) + self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], sorted(PQ_SUITES)) + + # The report step reads the evidence from BASE_BUILD_DIR, which a step + # sets from RUNNER_TEMP: the runner context is unavailable in job env. + self.assertNotIn("BASE_BUILD_DIR", job["env"]) + set_dir = svc.find_step(job, "Set build directory") + self.assertLess(svc.step_index(job, "Set build directory"), svc.step_index(job, "CI script")) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + github_env_file = Path(tmp) / "github-env" + github_env_file.touch() + step_env = svc.step_env(workflow, job, set_dir, context) + step_env.update({"RUNNER_TEMP": "/runner/temp", "GITHUB_ENV": str(github_env_file)}) + result = svc.run_step(fixture, set_dir, step_env, fixture.root, svc.github_env_for(context, MACOS_JOB, workflow["name"])) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(github_env_file.read_text(encoding="utf8"), "BASE_BUILD_DIR=/runner/temp/build\n") + + # The architecture check refuses anything but arm64. + step = svc.find_step(job, "Require Apple Silicon") + self.assertLess(svc.step_index(job, "Require Apple Silicon"), svc.step_index(job, "CI script")) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + stubs = Path(tmp) + (stubs / "uname").write_text('#!/bin/sh\necho "$STUB_MACHINE"\n', encoding="utf8") + (stubs / "uname").chmod(0o755) + for machine, ok in (("arm64", True), ("x86_64", False)): + with self.subTest(machine=machine): + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"PATH": f"{stubs}:{os.environ['PATH']}", "STUB_MACHINE": machine}) + result = svc.run_step(fixture, step, step_env, fixture.root, svc.github_env_for(context, MACOS_JOB, workflow["name"])) + self.assertEqual(result.returncode == 0, ok, result.stderr) + if not ok: + self.assertIn("must run on arm64", result.stderr) + + # Evidence is reported exactly as for the matrix jobs. + report = svc.find_step(job, REPORT_STEP) + self.assertEqual(report.get("if"), "${{ always() }}") + self.assertEqual(report["run"], svc.find_step(workflow["jobs"][PQ_UNIT_JOB], REPORT_STEP)["run"]) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + build = Path(tmp) + (build / "ctest-evidence").mkdir() + (build / "ctest-evidence" / "summary.md").write_text("| mlkem_tests | passed | 0.5 |\n", encoding="utf8") + step_env = svc.step_env(workflow, job, report, context) + step_env["BASE_BUILD_DIR"] = str(build) + result = svc.run_step(fixture, report, step_env, fixture.root, svc.github_env_for(context, MACOS_JOB, workflow["name"])) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("## macOS arm64 unit tests", result.summary) + self.assertIn("native AArch64", result.summary) + self.assertIn("| mlkem_tests | passed | 0.5 |", result.summary) + + def test_contexts_are_available_where_used(self) -> None: + """Mirror actionlint's context-availability rule for the places these workflows use expressions. + + GitHub refuses to load a workflow that reads, say, runner in a job's env, + while this file's expression evaluator would render it as empty. + """ + expression = re.compile(r"\$\{\{(.*?)\}\}", re.S) + root = re.compile(r"(? None: block = shell_block(TEST_SCRIPT.read_text(encoding="utf8"), "CI_BUILD_TARGET=${CI_BUILD_TARGET:-all}", ")") for target, expected in (("test_bitcoin", ["test_bitcoin"]), ("", ["all", "install"])): @@ -858,7 +991,7 @@ def test_build_target_reaches_both_build_commands(self) -> None: def test_ctest_selection_reaches_listing_run_and_evidence(self) -> None: block = shell_block(TEST_SCRIPT.read_text(encoding="utf8"), 'if [ "$RUN_UNIT_TESTS" = "true" ]; then', "fi") - regex = "^(bip324_tests|net_tests)$" + regex = "^(mlkem_tests|bip324_tests|net_tests)$" expected = " ".join(AARCH64_SUITES) run = run_test_script_block(block, {"CTEST_REGEX": regex, "CTEST_EXPECTED_SUITES": expected}) @@ -880,12 +1013,12 @@ def test_ctest_selection_reaches_listing_run_and_evidence(self) -> None: failures = { "skipped suite": ({"CTEST_REGEX": regex, "CTEST_EXPECTED_SUITES": expected, "STUB_OUTCOMES": json.dumps({"bip324_tests": "notrun"})}, "bip324_tests: skipped"), - "regex selects too little": ({"CTEST_REGEX": "^(net_tests)$", "CTEST_EXPECTED_SUITES": expected}, + "regex selects too little": ({"CTEST_REGEX": "^(mlkem_tests|net_tests)$", "CTEST_EXPECTED_SUITES": expected}, "did not select expected suite(s): bip324_tests"), - "regex selects too much": ({"CTEST_REGEX": "^bip324_tests|^net_tests$", "CTEST_EXPECTED_SUITES": expected}, + "regex selects too much": ({"CTEST_REGEX": "^mlkem_tests$|^bip324_tests|^net_tests$", "CTEST_EXPECTED_SUITES": expected}, "not expected: bip324_tests_extra"), - "suite missing from the build": ({"CTEST_REGEX": "^(bip324_tests|net_tests|mlkem_tests)$", - "CTEST_EXPECTED_SUITES": expected + " mlkem_tests"}, "mlkem_tests"), + "suite missing from the build": ({"CTEST_REGEX": "^(mlkem_tests|bip324_tests|net_tests|pq_records_tests)$", + "CTEST_EXPECTED_SUITES": expected + " pq_records_tests"}, "pq_records_tests"), "range and regex together": ({"CTEST_REGEX": regex, "CTEST_INCLUDE_RANGE": "1,2"}, "set only one"), "nothing selected": ({"CTEST_REGEX": "^nothing$"}, "No tests were found"), } @@ -929,7 +1062,7 @@ def test_report_step_publishes_source_and_suites(self) -> None: report = svc.run_step(fixture, step, env, fixture.root, github_env) self.assertEqual(report.returncode, 0, report.stderr) self.assertIn(f"- source: {fixture.manual_sha} (refs/heads/{fixture.manual_branch})", report.summary) - self.assertIn("Pre-library phase", report.summary) + self.assertIn("mlkem_tests, bip324_tests and net_tests on AArch64", report.summary) self.assertIn("| bip324_tests | passed | 0.5 |", report.summary) def test_required_gate_runs_the_contracts(self) -> None: diff --git a/ci/test/00_setup_env_mac_native_pq.sh b/ci/test/00_setup_env_mac_native_pq.sh new file mode 100755 index 0000000000..8de8148b3e --- /dev/null +++ b/ci/test/00_setup_env_mac_native_pq.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +# The macos-arm64-unit job of .github/workflows/ci-pq.yml: build only the +# unit-test binary natively on an Apple Silicon runner, without Docker, and run +# the post-quantum suites. This is the target that runs native AArch64 ML-KEM. +export CONTAINER_NAME="ci_mac_native_pq" # macos does not use a container, but the env var is needed for logging +export CMAKE_GENERATOR="Ninja" +export CI_OS_NAME="macos" +export NO_DEPENDS=1 +export OSX_SDK="" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=false +export CI_BUILD_TARGET=test_bitcoin +# The suites this job must run; ctest_evidence.py fails the job unless each was +# selected, executed and passed. CTEST_REGEX is derived from the same list. +export CTEST_EXPECTED_SUITES="mlkem_tests bip324_tests net_tests" +export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" +# WITH_MLKEM_NATIVE=ON makes losing the native AArch64 backend a configure error +# instead of a quiet fall back to portable C. +export BITCOIN_CONFIG="-DBUILD_GUI=OFF -DENABLE_WALLET=OFF -DENABLE_IPC=OFF -DWITH_ZMQ=OFF -DREDUCE_EXPORTS=ON -DWITH_MLKEM_NATIVE=ON" diff --git a/ci/test/00_setup_env_native_aarch64_pq.sh b/ci/test/00_setup_env_native_aarch64_pq.sh index ddb42f03c7..2b50cf8a83 100755 --- a/ci/test/00_setup_env_native_aarch64_pq.sh +++ b/ci/test/00_setup_env_native_aarch64_pq.sh @@ -20,7 +20,7 @@ export RUN_FUNCTIONAL_TESTS=false export CI_BUILD_TARGET=test_bitcoin # The suites this job must run; ctest_evidence.py fails the job unless each was # selected, executed and passed. CTEST_REGEX is derived from the same list. -export CTEST_EXPECTED_SUITES="bip324_tests net_tests" +export CTEST_EXPECTED_SUITES="mlkem_tests bip324_tests net_tests" export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" # -Wno-psabi silences GCC's AArch64 "parameter passing ... changed" notes. export BITCOIN_CONFIG="\ diff --git a/ci/test/00_setup_env_s390x_unit.sh b/ci/test/00_setup_env_s390x_unit.sh new file mode 100755 index 0000000000..5f28461617 --- /dev/null +++ b/ci/test/00_setup_env_s390x_unit.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +# shellcheck source=ci/test/00_setup_env_base_image.sh +source "$( dirname "${BASH_SOURCE[0]}" )/00_setup_env_base_image.sh" + +# The s390x-unit job of .github/workflows/ci-pq.yml: cross-compile only the +# unit-test binary for big-endian s390x on the x64 runner, with depends, and run +# the post-quantum suites under qemu-s390x. ctest adds the emulator to each test +# command, so only the tests run emulated; test_qbit is never run directly. +export HOST=s390x-linux-gnu +export CONTAINER_NAME=ci_s390x_pq +ci_set_base_image_name_tag "ubuntu:24.04" +export PACKAGES="g++-s390x-linux-gnu qemu-user" +export DEP_OPTS="NO_QT=1 NO_QR=1 NO_ZMQ=1 NO_WALLET=1 NO_USDT=1 NO_IPC=1" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=false +export CI_BUILD_TARGET=test_bitcoin +# The suites this job must run; ctest_evidence.py fails the job unless each was +# selected, executed and passed. CTEST_REGEX is derived from the same list. +export CTEST_EXPECTED_SUITES="mlkem_tests bip324_tests net_tests" +export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" +export BITCOIN_CONFIG="\ + -DBUILD_GUI=OFF -DENABLE_WALLET=OFF -DENABLE_IPC=OFF -DWITH_ZMQ=OFF -DREDUCE_EXPORTS=ON \ + -DCMAKE_CROSSCOMPILING_EMULATOR='qemu-s390x;-L;/usr/s390x-linux-gnu' \ +" From bf3dc798f15763b5b9f6c8c657ee7a26c6ad6389 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Mon, 5 Oct 2026 13:37:13 -0400 Subject: [PATCH 08/20] ci: gate ML-KEM and transport changes on the aarch64 PQ job The merge-profile classifier gains a pq_aarch64_required output, true when a change touches the ML-KEM library (src/mlkem-native, src/crypto/mlkem*, cmake/mlkem-native.cmake), the v2 transport (src/bip324*, src/net.h, src/net.cpp), the CPU feature checks, the PQ workflow or its env files, or the PQ test suites and vectors. A path it cannot read also requires the job. Full Validation calls ci-pq.yml with jobs: aarch64-unit on the change itself (a pull request's merge commit) when the output is true. The gate accepts a skipped job only when the classifier says it is not required; a required job that was skipped, failed or cancelled, or a missing or unreadable classifier output, fails it. The changed-file lists that feed the classifier, in Full Validation and in the Required Merge Gate, use --no-renames, so moving a file out of a gated path still names it. ci/checks/test_pq_merge_gate_contract.py runs the real classifier, job condition, ci-pq selection step and gate script: a pull request touching src/bip324.cpp requires and runs the job, while docs-only and other source pull requests skip it and pass, and a rename out of a gated path still requires it. The Required Merge Gate runs it with the other PQ CI contracts. Part of #184 (part 3, #190, C3). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 43 ++- .github/workflows/required-merge-gate.yml | 12 +- ci/checks/classify_merge_profile.py | 62 +++++ ci/checks/test_classify_merge_profile.py | 76 ++++++ ci/checks/test_pq_merge_gate_contract.py | 307 ++++++++++++++++++++++ 5 files changed, 491 insertions(+), 9 deletions(-) create mode 100755 ci/checks/test_pq_merge_gate_contract.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4db0ffa7b..651ee0dd49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,7 @@ jobs: profile: ${{ steps.classify.outputs.profile }} release_policy_only: ${{ steps.classify.outputs.release_policy_only }} source_validation_required: ${{ steps.classify.outputs.source_validation_required }} + pq_aarch64_required: ${{ steps.classify.outputs.pq_aarch64_required }} steps: - name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 @@ -96,7 +97,7 @@ jobs: echo "::error::Pull request base SHA is unavailable." exit 1 fi - git diff --name-only "${BASE_SHA}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BASE_SHA}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "merge_group" ]]; then base_branch="${BASE_REF#refs/heads/}" if [[ -z "${base_branch}" ]]; then @@ -104,15 +105,15 @@ jobs: exit 1 fi git fetch --no-tags --prune origin "+refs/heads/${base_branch}:refs/remotes/origin/${base_branch}" - git diff --name-only "origin/${base_branch}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "origin/${base_branch}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "push" ]]; then if [[ -n "${BEFORE_SHA}" && ! "${BEFORE_SHA}" =~ ^0+$ ]]; then - git diff --name-only "${BEFORE_SHA}..HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BEFORE_SHA}..HEAD" > "${changed_files}" else git fetch --no-tags --prune origin "${DEFAULT_BRANCH}" merge_base="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD 2>/dev/null || true)" if [[ -n "${merge_base}" ]]; then - git diff --name-only "${merge_base}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${merge_base}...HEAD" > "${changed_files}" else git diff-tree --no-commit-id --name-only -r HEAD > "${changed_files}" fi @@ -1071,6 +1072,18 @@ jobs: path: | build/doc/rpc/** + # Changes to the ML-KEM library, the v2 transport, or their build and CI + # wiring (ci/checks/classify_merge_profile.py) must pass the aarch64 job of + # the PQ workflow on the change itself: for a pull request, its merge commit. + pq-aarch64: + name: 'PQ aarch64 unit tests' + needs: classify-changes + if: ${{ needs.classify-changes.outputs.pq_aarch64_required == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} + uses: ./.github/workflows/ci-pq.yml + with: + source_ref: ${{ github.sha }} + jobs: aarch64-unit + full-validation-gate: name: 'Full Validation Gate' runs-on: ubuntu-24.04 @@ -1084,6 +1097,7 @@ jobs: - lint - public-docs-lint - rpc-docs + - pq-aarch64 if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} steps: - name: Require trusted label for fork PRs @@ -1106,6 +1120,8 @@ jobs: LINT_RESULT: ${{ needs.lint.result }} PUBLIC_DOCS_LINT_RESULT: ${{ needs.public-docs-lint.result }} RPC_DOCS_RESULT: ${{ needs.rpc-docs.result }} + PQ_AARCH64_REQUIRED: ${{ needs.classify-changes.outputs.pq_aarch64_required }} + PQ_AARCH64_RESULT: ${{ needs.pq-aarch64.result }} run: | set -euo pipefail @@ -1124,6 +1140,25 @@ jobs: check_result "classify validation profile" "${CLASSIFY_CHANGES_RESULT}" echo "CI profile: ${CI_PROFILE:-unavailable}" + # The aarch64 PQ job may be skipped only when the classifier says this + # change does not need it. + case "${PQ_AARCH64_REQUIRED}" in + true) + check_result "PQ aarch64 unit tests" "${PQ_AARCH64_RESULT}" + ;; + false) + echo "PQ aarch64 unit tests: ${PQ_AARCH64_RESULT} (not required for this change)" + if [[ "${PQ_AARCH64_RESULT}" != "skipped" && "${PQ_AARCH64_RESULT}" != "success" ]]; then + echo "::error title=Full Validation failed::PQ aarch64 unit tests result was ${PQ_AARCH64_RESULT}." + failed=1 + fi + ;; + *) + echo "::error title=Full Validation failed::The classifier did not report whether the PQ aarch64 job is required (pq_aarch64_required='${PQ_AARCH64_REQUIRED}')." + failed=1 + ;; + esac + if [[ "${SOURCE_VALIDATION_REQUIRED}" != "true" ]]; then echo "Validation profile ${VALIDATION_PROFILE} does not require full source validation." exit "${failed}" diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index e9e4008c2a..0c5f1b9b0f 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -89,7 +89,7 @@ jobs: echo "::error::Pull request base SHA is unavailable." exit 1 fi - git diff --name-only "${BASE_SHA}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BASE_SHA}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "merge_group" ]]; then base_branch="${BASE_REF#refs/heads/}" if [[ -z "${base_branch}" ]]; then @@ -97,15 +97,15 @@ jobs: exit 1 fi git fetch --no-tags --prune origin "+refs/heads/${base_branch}:refs/remotes/origin/${base_branch}" - git diff --name-only "origin/${base_branch}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "origin/${base_branch}...HEAD" > "${changed_files}" elif [[ "${EVENT_NAME}" == "push" ]]; then if [[ -n "${BEFORE_SHA}" && ! "${BEFORE_SHA}" =~ ^0+$ ]]; then - git diff --name-only "${BEFORE_SHA}..HEAD" > "${changed_files}" + git diff --name-only --no-renames "${BEFORE_SHA}..HEAD" > "${changed_files}" else git fetch --no-tags --prune origin "${DEFAULT_BRANCH}" merge_base="$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD 2>/dev/null || true)" if [[ -n "${merge_base}" ]]; then - git diff --name-only "${merge_base}...HEAD" > "${changed_files}" + git diff --name-only --no-renames "${merge_base}...HEAD" > "${changed_files}" else git diff-tree --no-commit-id --name-only -r HEAD > "${changed_files}" fi @@ -143,12 +143,14 @@ jobs: # Job selection, source resolution and unit-test evidence of the PQ and # nightly workflows, which run outside pull requests, the nightly - # workflow's scheduled source, and the PQ canary tools. + # workflow's scheduled source, the PQ canary tools, and the aarch64 PQ job + # that Full Validation requires for ML-KEM and transport changes. - name: Test PQ CI workflow contract run: | /usr/bin/python3 ci/checks/test_ci_pq_contract.py -v /usr/bin/python3 ci/checks/test_scheduled_validation_contract.py -v /usr/bin/python3 contrib/pq-canary/test/test_pq_canary.py -v + /usr/bin/python3 ci/checks/test_pq_merge_gate_contract.py -v - name: Classify validation profile id: classify diff --git a/ci/checks/classify_merge_profile.py b/ci/checks/classify_merge_profile.py index b1d32714a9..bb3fa82fb7 100755 --- a/ci/checks/classify_merge_profile.py +++ b/ci/checks/classify_merge_profile.py @@ -55,6 +55,57 @@ "contrib/keys/operator-keys/", "contrib/guix/repo-templates/qbit-guix.sigs/operator-keys/", ) +# Changes to the ML-KEM library, the v2 transport, or their build and CI wiring +# must pass the aarch64 job of .github/workflows/ci-pq.yml before they merge. +# ci/checks/test_pq_merge_gate_contract.py fails if a CMake file, workflow or +# ci/ file that mentions ML-KEM is missing here. +PQ_AARCH64_FILES = frozenset( + { + # The build: the WITH_MLKEM_NATIVE option, the library and its link + # into bitcoin_node, the run-time CPU detection, the configure header, + # and the tests, fuzz targets and benchmarks that exercise it. + "CMakeLists.txt", + "cmake/bitcoin-build-config.h.in", + "cmake/mlkem-native.cmake", + "src/CMakeLists.txt", + "src/bench/CMakeLists.txt", + "src/test/CMakeLists.txt", + "src/test/fuzz/CMakeLists.txt", + "src/util/CMakeLists.txt", + # The jobs and the gate: this classifier, the workflows that run it and + # the PQ jobs, the scripts and evidence check of those jobs, and the + # tests of that wiring. + ".github/workflows/ci-pq.yml", + ".github/workflows/ci.yml", + ".github/workflows/core-checks.yml", + ".github/workflows/required-merge-gate.yml", + "ci/checks/classify_merge_profile.py", + "ci/checks/ctest_evidence.py", + "ci/checks/test_ci_pq_contract.py", + "ci/checks/test_classify_merge_profile.py", + "ci/checks/test_mlkem_build_policy.py", + "ci/checks/test_pq_merge_gate_contract.py", + "ci/test/00_setup_env_mac_native_pq.sh", + "ci/test/00_setup_env_native_aarch64_pq.sh", + "ci/test/00_setup_env_s390x_unit.sh", + "ci/test/03_test_script.sh", + # The sources and tests. + "src/compat/cpuid.h", + "src/net.cpp", + "src/net.h", + "src/test/bip324_tests.cpp", + "src/test/data/mlkem1024_vectors.json", + "src/test/data/pq_transport_vectors.json", + "src/test/mlkem_tests.cpp", + "src/test/net_tests.cpp", + } +) +PQ_AARCH64_PREFIXES = ( + "src/bip324", + "src/compat/cpu_features.", + "src/crypto/mlkem", + "src/mlkem-native/", +) @dataclass(frozen=True) @@ -105,6 +156,11 @@ def public_docs_only(self) -> bool: def github_metadata_only(self) -> bool: return self.profile == GITHUB_METADATA_PROFILE + @property + def pq_aarch64_required(self) -> bool: + """Whether the aarch64 PQ job must pass. A path that cannot be read requires it.""" + return bool(self.invalid_paths) or any(is_pq_aarch64_path(path) for path in self.paths) + def normalize_path(path: str) -> str | None: path = path.strip() @@ -147,6 +203,10 @@ def is_github_metadata_path(path: str) -> bool: return path in GITHUB_METADATA_FILES or path.startswith(GITHUB_METADATA_PREFIXES) +def is_pq_aarch64_path(path: str) -> bool: + return path in PQ_AARCH64_FILES or path.startswith(PQ_AARCH64_PREFIXES) + + def classify_paths(paths: list[str] | tuple[str, ...]) -> Classification: normalized: list[str] = [] invalid: list[str] = [] @@ -223,6 +283,7 @@ def github_outputs(classification: Classification) -> dict[str, str]: "touched_rpc_docs": bool_output(bool(classification.rpc_docs_paths)), "touched_public_docs": bool_output(bool(classification.public_docs_paths)), "touched_github_metadata": bool_output(bool(classification.github_metadata_paths)), + "pq_aarch64_required": bool_output(classification.pq_aarch64_required), } @@ -236,6 +297,7 @@ def describe_classification(classification: Classification) -> str: lines = [ f"validation_profile={classification.profile}", f"changed_count={len(classification.paths)}", + f"pq_aarch64_required={bool_output(classification.pq_aarch64_required)}", ] if classification.release_policy_paths: diff --git a/ci/checks/test_classify_merge_profile.py b/ci/checks/test_classify_merge_profile.py index b2e3824651..16521b8221 100755 --- a/ci/checks/test_classify_merge_profile.py +++ b/ci/checks/test_classify_merge_profile.py @@ -242,6 +242,82 @@ def test_github_outputs_report_github_metadata_profile(self) -> None: self.assertEqual(outputs["source_validation_required"], "false") self.assertEqual(outputs["touched_github_metadata"], "true") + def test_pq_aarch64_job_required_for_mlkem_and_transport_paths(self) -> None: + required = [ + "src/mlkem-native/mlkem/src/kem.c", + "src/crypto/mlkem.cpp", + "src/crypto/mlkem_config.h", + "cmake/mlkem-native.cmake", + "src/bip324.cpp", + "src/bip324.h", + "src/bip324_pq.cpp", + "src/bip324_pq.h", + "src/net.h", + "src/net.cpp", + "src/compat/cpuid.h", + "src/compat/cpu_features.h", + "src/compat/cpu_features.cpp", + ".github/workflows/ci-pq.yml", + "ci/test/00_setup_env_native_aarch64_pq.sh", + "ci/test/00_setup_env_s390x_unit.sh", + "ci/test/00_setup_env_mac_native_pq.sh", + "src/test/mlkem_tests.cpp", + "src/test/bip324_tests.cpp", + "src/test/net_tests.cpp", + "src/test/data/pq_transport_vectors.json", + "src/test/data/mlkem1024_vectors.json", + # Build wiring. + "CMakeLists.txt", + "src/CMakeLists.txt", + "src/util/CMakeLists.txt", + "src/test/CMakeLists.txt", + "src/test/fuzz/CMakeLists.txt", + "src/bench/CMakeLists.txt", + "cmake/bitcoin-build-config.h.in", + # Gate and CI wiring, and its tests. + ".github/workflows/ci.yml", + ".github/workflows/core-checks.yml", + ".github/workflows/required-merge-gate.yml", + "ci/checks/classify_merge_profile.py", + "ci/checks/ctest_evidence.py", + "ci/checks/test_ci_pq_contract.py", + "ci/checks/test_classify_merge_profile.py", + "ci/checks/test_mlkem_build_policy.py", + "ci/checks/test_pq_merge_gate_contract.py", + "ci/test/03_test_script.sh", + ] + for path in required: + with self.subTest(path=path): + classification = self.classify([path]) + self.assertTrue(classification.pq_aarch64_required) + self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_aarch64_required"], "true") + self.assertEqual(classification.profile, classify_merge_profile.SOURCE_PROFILE) + # One matching path among others is enough. + self.assertTrue(self.classify(["doc/user/README.md", "src/bip324.cpp"]).pq_aarch64_required) + + def test_pq_aarch64_job_not_required_elsewhere(self) -> None: + for paths in ( + ["src/wallet/wallet.cpp"], + ["src/netbase.cpp", "src/net_processing.cpp", "src/compat/compat.h"], + ["src/test/net_peer_eviction_tests.cpp"], + ["ci/test/00_setup_env_native_asan.sh", ".github/workflows/rpc-perf-manual.yml"], + # Wiring shared by every CI job, and ML-KEM tooling the aarch64 job never runs. + ["ci/test/01_base_install.sh", "ci/test/02_run_container.py", ".github/actions/configure-docker/action.yml"], + ["src/crypto/CMakeLists.txt", "src/test/cpu_features_tests.cpp"], + ["test/lint/mlkem-native-check.sh", "contrib/devtools/mlkem-native.pin", "contrib/devtools/update-mlkem-native.sh"], + ["doc/user/README.md"], + RPC_DOCS_PATHS, + PUBLIC_DOCS_PATHS, + ): + with self.subTest(paths=paths): + classification = self.classify(paths) + self.assertFalse(classification.pq_aarch64_required) + self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_aarch64_required"], "false") + + def test_pq_aarch64_job_required_for_unreadable_paths(self) -> None: + self.assertTrue(self.classify(["../src/bip324.cpp"]).pq_aarch64_required) + self.assertTrue(self.classify(["/abs/path"]).pq_aarch64_required) + def test_require_release_policy_only_cli_rejects_outside_path(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: changed_files = Path(tmpdir) / "changed-files.txt" diff --git a/ci/checks/test_pq_merge_gate_contract.py b/ci/checks/test_pq_merge_gate_contract.py new file mode 100755 index 0000000000..177a842a49 --- /dev/null +++ b/ci/checks/test_pq_merge_gate_contract.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026-present The qbit core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. +"""Contract for the aarch64 PQ job in Full Validation (#184 part 3, work item C3). + +Pull requests that touch the ML-KEM library, the v2 transport, or their build +and CI wiring must pass the aarch64-unit job of ci-pq.yml before they merge; +others skip it. These tests run the real pieces of that chain: + +* ci/checks/classify_merge_profile.py, as the classify-changes job runs it, + reports pq_aarch64_required; +* the pq-aarch64 job of ci.yml calls ci-pq.yml with jobs aarch64-unit on the + change's own commit (a pull request's merge commit) exactly when that output + is true, and ci-pq.yml's selection step then selects only aarch64-unit; +* full-validation-gate's shell accepts a skipped job only when the classifier + says it is not required, and fails on a missing classifier output; +* the changed-file lists that feed the classifier name both sides of a rename; +* every CMake file, workflow or ci/ file that mentions ML-KEM classifies as + requiring the job, so new wiring cannot be missed silently. + +GitHub itself is not called. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import test_scheduled_validation_contract as svc + +REPO_ROOT = svc.REPO_ROOT +CI_WORKFLOW = svc.WORKFLOW_DIR / "ci.yml" +PQ_WORKFLOW = svc.WORKFLOW_DIR / "ci-pq.yml" +GATE_WORKFLOW = svc.WORKFLOW_DIR / "required-merge-gate.yml" +CLASSIFIER = REPO_ROOT / "ci" / "checks" / "classify_merge_profile.py" +CLASSIFY_JOB = "classify-changes" +PQ_JOB = "pq-aarch64" +GATE_JOB = "full-validation-gate" +GATE_STEP = "Check validation results" +AARCH64_JOB = "aarch64-unit" +OUTPUT = "pq_aarch64_required" +MERGE_SHA = "1234567890abcdef1234567890abcdef12345678" +HEAD_SHA = "fedcba0987654321fedcba0987654321fedcba09" +CHANGED_FILES_STEP = "Write changed file list" +# Files that can wire ML-KEM into the build or CI: CMake files, workflows and +# composite actions, and everything under ci/. +WIRING_PATH_RE = re.compile(r"(^|/)CMakeLists[.]txt$|[.]cmake([.]in)?$|^cmake/|^[.]github/(workflows|actions)/|^ci/") +MLKEM_RE = re.compile(rb"ml[-_]?kem", re.IGNORECASE) + +MATCHING_PR = ["src/bip324.cpp"] +DOCS_ONLY_PR = ["doc/user/README.md"] +OTHER_SOURCE_PR = ["src/wallet/wallet.cpp"] + +# Every other job the gate checks, passing. +OTHER_GATE_RESULTS = { + "RUNNERS_RESULT": "success", + "WINDOWS_NATIVE_DLL_RESULT": "success", + "WINDOWS_CROSS_RESULT": "success", + "WINDOWS_NATIVE_TEST_RESULT": "success", + "CI_MATRIX_RESULT": "success", + "LINT_RESULT": "success", + "PUBLIC_DOCS_LINT_RESULT": "success", + "RPC_DOCS_RESULT": "success", +} + + +def classify(paths: list[str]) -> dict[str, str]: + """Run the classifier the way classify-changes does and return its GitHub outputs.""" + with tempfile.TemporaryDirectory() as tmp: + changed = Path(tmp) / "changed-files.txt" + changed.write_text("".join(f"{path}\n" for path in paths), encoding="utf8") + output = Path(tmp) / "github-output" + output.touch() + completed = subprocess.run( + [sys.executable, str(CLASSIFIER), "--changed-files", str(changed), "--github-output", str(output)], + cwd=REPO_ROOT, text=True, capture_output=True, check=False, + ) + assert completed.returncode == 0, completed.stderr + return svc.parse_key_values(output.read_text(encoding="utf8")) + + +def pull_request_context(outputs: dict[str, str], *, draft: bool = False) -> dict[str, Any]: + return { + "github": { + "event_name": "pull_request", + "sha": MERGE_SHA, + # The pull request's head differs from the merge commit the job must validate. + "event": {"pull_request": {"draft": draft, "head": {"sha": HEAD_SHA}}}, + }, + "needs": {CLASSIFY_JOB: {"outputs": outputs, "result": "success"}}, + } + + +class PQMergeGateContractTest(unittest.TestCase): + ci: dict[str, Any] + pq: dict[str, Any] + + @classmethod + def setUpClass(cls) -> None: + cls.ci = svc.load_workflow(CI_WORKFLOW) + cls.pq = svc.load_workflow(PQ_WORKFLOW) + + def job_runs(self, context: dict[str, Any]) -> bool: + return svc._truthy(svc.render(self.ci["jobs"][PQ_JOB]["if"], context)) + + def run_gate(self, **env: str) -> subprocess.CompletedProcess[str]: + step = svc.find_step(self.ci["jobs"][GATE_JOB], GATE_STEP) + full_env = {"PATH": os.environ["PATH"], "LC_ALL": "C", "CLASSIFY_CHANGES_RESULT": "success", + "VALIDATION_PROFILE": "source", "SOURCE_VALIDATION_REQUIRED": "true", "CI_PROFILE": "internal"} + full_env.update(OTHER_GATE_RESULTS) + full_env.update(env) + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "gate.sh" + script.write_text(step["run"], encoding="utf8") + return subprocess.run(["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)], + env=full_env, text=True, capture_output=True, check=False) + + def select_pq_jobs(self, jobs: str) -> dict[str, str]: + """Run ci-pq.yml's selection step with the jobs input the caller passes.""" + step = svc.find_step(self.pq["jobs"][svc.RESOLVER_JOB], "Select jobs") + env = {key: str(value) for key, value in step["env"].items() if key != "JOBS_INPUT"} + env.update({"PATH": os.environ["PATH"], "JOBS_INPUT": jobs}) + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "select.py" + script.write_text(step["run"], encoding="utf8") + output = Path(tmp) / "github-output" + output.touch() + env["GITHUB_OUTPUT"] = str(output) + env["GITHUB_STEP_SUMMARY"] = os.devnull + completed = subprocess.run([sys.executable, str(script)], env=env, text=True, capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + return svc.parse_key_values(output.read_text(encoding="utf8")) + + # -- wiring ---------------------------------------------------------------------- + + def test_classify_changes_exposes_the_output(self) -> None: + job = self.ci["jobs"][CLASSIFY_JOB] + self.assertEqual(job["outputs"][OUTPUT], "${{ steps.classify.outputs." + OUTPUT + " }}") + classify_step = svc.find_step(job, "Classify validation profile") + self.assertIn("ci/checks/classify_merge_profile.py", classify_step["run"]) + self.assertIn('--github-output "$GITHUB_OUTPUT"', classify_step["run"]) + + def test_pq_job_calls_only_aarch64_on_the_change_itself(self) -> None: + job = self.ci["jobs"][PQ_JOB] + self.assertEqual(job["uses"], "./.github/workflows/ci-pq.yml") + self.assertEqual(job["needs"], CLASSIFY_JOB) + context = pull_request_context({OUTPUT: "true"}) + rendered = {key: svc._to_text(svc.render(value, context)) for key, value in job["with"].items()} + self.assertEqual(rendered, {"source_ref": MERGE_SHA, "jobs": AARCH64_JOB}) + self.assertEqual(set(job["with"]), set(self.pq["on"]["workflow_call"]["inputs"])) + + outputs = self.select_pq_jobs(rendered["jobs"]) + self.assertEqual([entry["job"] for entry in json.loads(outputs["matrix"])["include"]], [AARCH64_JOB]) + self.assertEqual(outputs["run_macos_arm64_unit"], "false") + + def test_required_merge_gate_runs_this_contract(self) -> None: + gate = svc.load_workflow(GATE_WORKFLOW) + commands = "\n".join(step.get("run", "") for step in gate["jobs"][CLASSIFY_JOB]["steps"]) + self.assertIn(f"ci/checks/{Path(__file__).name}", commands) + + def test_changed_files_name_both_sides_of_a_rename(self) -> None: + """A rename out of a gated path must still require the job: git diff lists only the new path by default.""" + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) / "repo" + repo.mkdir() + git_env = {"PATH": os.environ["PATH"], "HOME": tmp, "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + + def git(*args: str) -> str: + return subprocess.run(["git", "-c", "commit.gpgsign=false", *args], cwd=repo, env=git_env, text=True, + capture_output=True, check=True).stdout.strip() + + git("init", "-q") + (repo / "src" / "crypto").mkdir(parents=True) + (repo / "src" / "crypto" / "mlkem.cpp").write_text("".join(f"line {i}\n" for i in range(50)), encoding="utf8") + git("add", "-A") + git("commit", "-q", "-m", "base") + base = git("rev-parse", "HEAD") + (repo / "src" / "crypto" / "kem").mkdir() + git("mv", "src/crypto/mlkem.cpp", "src/crypto/kem/kyber.cpp") + with open(repo / "src" / "crypto" / "kem" / "kyber.cpp", "a", encoding="utf8") as handle: + handle.write("edited\n") + git("commit", "-q", "-am", "rename") + self.assertEqual(git("diff", "--name-only", f"{base}...HEAD"), "src/crypto/kem/kyber.cpp", "git detects the rename") + + for workflow_path in (CI_WORKFLOW, GATE_WORKFLOW): + with self.subTest(workflow=workflow_path.name): + step = svc.find_step(svc.load_workflow(workflow_path)["jobs"][CLASSIFY_JOB], CHANGED_FILES_STEP) + output = Path(tmp) / "github-output" + output.write_text("", encoding="utf8") + env = dict(git_env, EVENT_NAME="pull_request", BASE_SHA=base, RUNNER_TEMP=tmp, GITHUB_OUTPUT=str(output)) + script = Path(tmp) / "step.sh" + script.write_text(step["run"], encoding="utf8") + completed = subprocess.run(["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)], + cwd=repo, env=env, text=True, capture_output=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + changed = Path(svc.parse_key_values(output.read_text(encoding="utf8"))["changed_files"]) + paths = changed.read_text(encoding="utf8").split() + self.assertEqual(sorted(paths), ["src/crypto/kem/kyber.cpp", "src/crypto/mlkem.cpp"]) + self.assertEqual(classify(paths)[OUTPUT], "true") + + def test_gate_reads_the_job_and_the_classifier(self) -> None: + gate = self.ci["jobs"][GATE_JOB] + self.assertIn(PQ_JOB, gate["needs"]) + self.assertIn("always()", gate["if"]) + env = svc.find_step(gate, GATE_STEP)["env"] + self.assertEqual(env["PQ_AARCH64_REQUIRED"], "${{ needs." + CLASSIFY_JOB + ".outputs." + OUTPUT + " }}") + self.assertEqual(env["PQ_AARCH64_RESULT"], "${{ needs." + PQ_JOB + ".result }}") + + def test_job_condition(self) -> None: + self.assertTrue(self.job_runs(pull_request_context({OUTPUT: "true"}))) + self.assertFalse(self.job_runs(pull_request_context({OUTPUT: "false"}))) + self.assertFalse(self.job_runs(pull_request_context({})), "a missing output never starts the job") + self.assertFalse(self.job_runs(pull_request_context({OUTPUT: "true"}, draft=True))) + push = pull_request_context({OUTPUT: "true"}) + push["github"] = {"event_name": "push", "sha": MERGE_SHA, "event": {}} + self.assertTrue(self.job_runs(push)) + + # -- the gate ---------------------------------------------------------------------- + + def test_gate_outcomes(self) -> None: + cases = [ + ("required and passed", "true", "success", True, ""), + ("required but skipped", "true", "skipped", False, "PQ aarch64 unit tests result was skipped"), + ("required but failed", "true", "failure", False, "PQ aarch64 unit tests result was failure"), + ("required but cancelled", "true", "cancelled", False, "PQ aarch64 unit tests result was cancelled"), + ("not required, skipped", "false", "skipped", True, ""), + ("not required, failed anyway", "false", "failure", False, "PQ aarch64 unit tests result was failure"), + ("classifier output missing", "", "skipped", False, "did not report whether the PQ aarch64 job is required"), + ("classifier output garbled", "yes", "success", False, "did not report whether the PQ aarch64 job is required"), + ] + for label, required, result, ok, message in cases: + with self.subTest(label): + run = self.run_gate(PQ_AARCH64_REQUIRED=required, PQ_AARCH64_RESULT=result) + self.assertEqual(run.returncode == 0, ok, run.stdout + run.stderr) + if message: + self.assertIn(message, run.stdout) + + # A missing output fails even a change that needs no source validation. + run = self.run_gate(PQ_AARCH64_REQUIRED="", PQ_AARCH64_RESULT="skipped", + SOURCE_VALIDATION_REQUIRED="false", VALIDATION_PROFILE="public-docs") + self.assertNotEqual(run.returncode, 0) + + # -- end to end ---------------------------------------------------------------------- + + def gate_for_pull_request(self, paths: list[str]) -> tuple[bool, subprocess.CompletedProcess[str]]: + """Classify, decide whether the job runs (passing when it does), and run the gate.""" + outputs = classify(paths) + runs = self.job_runs(pull_request_context(outputs)) + source = outputs["source_validation_required"] + others = OTHER_GATE_RESULTS if source == "true" else {key: "skipped" for key in OTHER_GATE_RESULTS} + gate = self.run_gate(PQ_AARCH64_REQUIRED=outputs[OUTPUT], PQ_AARCH64_RESULT="success" if runs else "skipped", + SOURCE_VALIDATION_REQUIRED=source, VALIDATION_PROFILE=outputs["profile"], **others) + return runs, gate + + def test_matching_pull_request_requires_and_runs_the_job(self) -> None: + runs, gate = self.gate_for_pull_request(MATCHING_PR) + self.assertTrue(runs) + self.assertEqual(gate.returncode, 0, gate.stdout + gate.stderr) + self.assertIn("PQ aarch64 unit tests: success", gate.stdout) + # Skipping it would fail the gate. + outputs = classify(MATCHING_PR) + self.assertNotEqual(self.run_gate(PQ_AARCH64_REQUIRED=outputs[OUTPUT], PQ_AARCH64_RESULT="skipped").returncode, 0) + + def test_other_pull_requests_skip_the_job_and_pass(self) -> None: + for paths in (DOCS_ONLY_PR, OTHER_SOURCE_PR): + with self.subTest(paths=paths): + runs, gate = self.gate_for_pull_request(paths) + self.assertFalse(runs) + self.assertEqual(gate.returncode, 0, gate.stdout + gate.stderr) + + +def repository_files() -> list[str]: + """Tracked files, and new ones git does not ignore.""" + completed = subprocess.run(["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], + cwd=REPO_ROOT, capture_output=True, check=False) + assert completed.returncode == 0, completed.stderr.decode(errors="replace") + return sorted({path for path in completed.stdout.decode("utf8").split("\0") if (REPO_ROOT / path).is_file()}) + + +class MlkemWiringContractTest(unittest.TestCase): + """The classifier knows every file that wires ML-KEM into the build or CI.""" + + def test_wiring_that_mentions_mlkem_requires_the_job(self) -> None: + wiring = [path for path in repository_files() + if WIRING_PATH_RE.search(path) and MLKEM_RE.search((REPO_ROOT / path).read_bytes())] + # The scan proves nothing unless it finds the wiring known today. + for known in ("CMakeLists.txt", "src/CMakeLists.txt", "cmake/mlkem-native.cmake", ".github/workflows/ci.yml", + "ci/checks/classify_merge_profile.py", "ci/test/03_test_script.sh"): + self.assertIn(known, wiring) + missing = [path for path in wiring if classify([path])[OUTPUT] != "true"] + self.assertEqual(missing, [], + "FAIL: these files mention ML-KEM but do not require the aarch64 PQ job\n" + "Cause: a change to only one of them would skip pq-aarch64, and Full Validation accepts the skip\n" + "Fix: add them to PQ_AARCH64_FILES in ci/checks/classify_merge_profile.py") + + +if __name__ == "__main__": + unittest.main() From ba911ee4c0500b4bb548aa9c03ad6506f52cb837 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 10:14:06 -0400 Subject: [PATCH 09/20] ci: run the PQ macOS arm64 job on macos-15 GitHub is retiring the macos-14 runner image: brownouts start on 2026-10-05 and the image is removed on 2026-11-02. Move the macos-arm64-unit job to macos-15, which is also Apple Silicon. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-pq.yml | 2 +- ci/checks/test_ci_pq_contract.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pq.yml b/.github/workflows/ci-pq.yml index e16273bda1..395c9072c2 100644 --- a/.github/workflows/ci-pq.yml +++ b/.github/workflows/ci-pq.yml @@ -354,7 +354,7 @@ jobs: name: macOS arm64 unit tests needs: resolve-source if: ${{ needs.resolve-source.outputs.run_macos_arm64_unit == 'true' }} - runs-on: macos-14 + runs-on: macos-15 timeout-minutes: 120 env: DANGER_RUN_CI_ON_HOST: 1 diff --git a/ci/checks/test_ci_pq_contract.py b/ci/checks/test_ci_pq_contract.py index bebd315bc7..078febfc7d 100755 --- a/ci/checks/test_ci_pq_contract.py +++ b/ci/checks/test_ci_pq_contract.py @@ -889,7 +889,7 @@ def test_macos_job_runs_natively_on_apple_silicon(self) -> None: workflow = self.workflows[PQ] job = workflow["jobs"][MACOS_JOB] self.assertNotIn(MACOS_JOB, [entry["job"] for entry in catalog(workflow)]) - self.assertEqual(job["runs-on"], "macos-14") + self.assertEqual(job["runs-on"], "macos-15") context = self.resolve(PQ, self.dispatch(PQ, jobs=MACOS_JOB)) env = svc.job_env(workflow, job, context) self.assertEqual(env["DANGER_RUN_CI_ON_HOST"], "1", "no Docker on the macOS host") From 8a1be40361e14d00624fd5712d85872bf0834d41 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 10:37:18 -0400 Subject: [PATCH 10/20] build: probe the ML-KEM target with the appended flags APPEND_CPPFLAGS and APPEND_CFLAGS reach mlkem_native's C and assembly compiles, but the architecture probes ran without them. A build that appends a target-changing flag such as -m32 was detected as x86_64, enabled the native backend, and then failed in mlkem_config.h. Run the probes with the appended flags, and probe afresh on every configure, since those flags may change between configures. Co-Authored-By: Claude Opus 5.5 --- ci/checks/test_mlkem_build_policy.py | 20 ++++++++++++++++++++ cmake/mlkem-native.cmake | 7 +++++++ 2 files changed, 27 insertions(+) diff --git a/ci/checks/test_mlkem_build_policy.py b/ci/checks/test_mlkem_build_policy.py index 7806d74ab7..28aac4bfc3 100755 --- a/ci/checks/test_mlkem_build_policy.py +++ b/ci/checks/test_mlkem_build_policy.py @@ -318,6 +318,26 @@ def test_compilers(self) -> None: self.assertIn("no BTI landing pads", combined) self.assertGreater(found, 0, "no usable C compiler found") + def test_append_flags_change_the_detected_target(self) -> None: + """APPEND_CPPFLAGS/APPEND_CFLAGS reach the library, so the target probes must see them: -m32 is not x86_64.""" + if host_arch() != "x86_64": + self.skipTest("needs an x86_64 host") + compiler = shutil.which("cc") + assert compiler is not None + if run([compiler, "-m32", "-x", "c", "-c", "-o", os.devnull, "-"], input="#include \nint x;\n").returncode != 0: + skip_missing_compiler(self, "cc-m32", "has no 32-bit headers") + name = "append_m32" + configured = self.configure(name, compiler, "-DWITH_MLKEM_NATIVE=AUTO", "-DAPPEND_CPPFLAGS=-m32", "-DAPPEND_CFLAGS=-m32") + output = configured.stdout + configured.stderr + self.assertEqual(configured.returncode, 0, output) + self.assertIn("SUMMARY portable", output, + f"FAIL: APPEND_CFLAGS=-m32 still detected x86_64\nCause: the target probes ran without the appended flags\nFix: mlkem_native_detect_arch in {MODULE}") + built = self.build(name) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + # A reconfigure without the flag probes again and goes back to native. + configured = self.configure(name, compiler, "-DWITH_MLKEM_NATIVE=AUTO", "-DAPPEND_CPPFLAGS=", "-DAPPEND_CFLAGS=") + self.assertIn("SUMMARY native", configured.stdout + configured.stderr, configured.stdout + configured.stderr) + def test_memory_sanitizer(self) -> None: compiler = shutil.which("cc") self.assertIsNotNone(compiler) diff --git a/cmake/mlkem-native.cmake b/cmake/mlkem-native.cmake index 299f2b3708..7fba4d7d54 100644 --- a/cmake/mlkem-native.cmake +++ b/cmake/mlkem-native.cmake @@ -77,6 +77,13 @@ endfunction() function(mlkem_native_detect_arch var) include(CheckCSourceCompiles) set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + # The library's compiles get APPEND_CPPFLAGS and APPEND_CFLAGS, which can + # change the target (-m32, say), so the probes must see them too. Probe + # afresh on every configure, as those flags may have changed. + string(STRIP "${CMAKE_REQUIRED_FLAGS} ${APPEND_CPPFLAGS} ${APPEND_CFLAGS}" CMAKE_REQUIRED_FLAGS) + unset(MLKEM_NATIVE_TARGET_X86_64 CACHE) + unset(MLKEM_NATIVE_TARGET_AARCH64_MACHO CACHE) + unset(MLKEM_NATIVE_TARGET_AARCH64_ELF CACHE) check_c_source_compiles(" #if !((defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64)) && !defined(__ILP32__)) #error not LP64 x86_64 From 92599fedb316bfe9443060f008530e13281c2fd4 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 12:39:22 -0400 Subject: [PATCH 11/20] ci: require the macOS arm64 and x86_64 PQ jobs on PQ changes No pull request ever built or ran ML-KEM's native code on macOS: the macos-arm64-unit job could only be dispatched, and nothing ran the x86_64 native code on an Intel Mac at all. Apple Silicon and Intel macOS are release targets with native code enabled. ci-pq.yml gains a macos-x86_64-unit job on GitHub's Intel image (macos-15-intel), sharing every step with macos-arm64-unit. Both build with WITH_MLKEM_NATIVE=ON, check the runner's architecture, and after the suites pass require the native backend by name from mlkem_tests/backend_selection, so a runner whose CPU lacks an extension cannot pass on portable C. Full Validation's PQ job, renamed pq-unit, now calls ci-pq.yml with aarch64-unit, macos-arm64-unit and macos-x86_64-unit, so all three are required exactly when the classifier says the change needs them. The classifier output is renamed pq_unit_required to match. On-demand runs accept the new job name; an empty jobs input runs it too. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-pq.yml | 103 +++++++++++++---- .github/workflows/ci.yml | 38 ++++--- .github/workflows/required-merge-gate.yml | 2 +- ci/checks/classify_merge_profile.py | 21 ++-- ci/checks/test_ci_pq_contract.py | 133 +++++++++++++++++----- ci/checks/test_classify_merge_profile.py | 22 ++-- ci/checks/test_pq_merge_gate_contract.py | 78 ++++++++----- ci/test/00_setup_env_mac_native_pq.sh | 11 +- 8 files changed, 284 insertions(+), 124 deletions(-) diff --git a/.github/workflows/ci-pq.yml b/.github/workflows/ci-pq.yml index 395c9072c2..20c63fc8da 100644 --- a/.github/workflows/ci-pq.yml +++ b/.github/workflows/ci-pq.yml @@ -9,10 +9,14 @@ # - on-demand runs go through main's PQ caller (C4, part 2 of #184), which takes # workflow_dispatch inputs source_ref and jobs and calls ci-pq.yml@1.x.x, for # example gh workflow run ci-maintained-pq.yml -f source_ref= -f jobs=aarch64-unit. -# That caller lands on main after this part merges. +# That caller lands on main after this part merges; +# - Full Validation (ci.yml) calls it with aarch64-unit, macos-arm64-unit and +# macos-x86_64-unit for every change that ci/checks/classify_merge_profile.py +# says needs them. # The workflow_dispatch trigger below stays for when this file is on main itself; # until then GitHub rejects gh workflow run ci-pq.yml. -# Jobs: aarch64-unit, s390x-unit (big-endian, emulated) and macos-arm64-unit. +# Jobs: aarch64-unit, s390x-unit (big-endian, emulated), macos-arm64-unit and +# macos-x86_64-unit. # The source is resolved once to a commit, every job checks out and verifies # that commit, and the summaries record it. @@ -28,7 +32,7 @@ on: type: string default: '' jobs: - description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit); empty runs all' + description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit, macos-x86_64-unit); empty runs all' required: false type: string default: '' @@ -40,7 +44,7 @@ on: type: string default: '' jobs: - description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit); empty runs all' + description: 'Comma-separated jobs to run (aarch64-unit, s390x-unit, macos-arm64-unit, macos-x86_64-unit); empty runs all' required: false type: string default: '' @@ -61,6 +65,7 @@ jobs: matrix_count: ${{ steps.select.outputs.matrix_count }} selected: ${{ steps.select.outputs.selected }} run_macos_arm64_unit: ${{ steps.select.outputs.run_macos_arm64_unit }} + run_macos_x86_64_unit: ${{ steps.select.outputs.run_macos_x86_64_unit }} steps: - name: Resolve source id: resolve @@ -185,7 +190,7 @@ jobs: } ] # Selectable jobs that are not matrix entries; each gets a run_ output. - NON_MATRIX_JOBS: 'macos-arm64-unit' + NON_MATRIX_JOBS: 'macos-arm64-unit macos-x86_64-unit' run: | # Turn the comma-separated jobs input into the matrix: empty selects # every job; an empty, unknown or repeated name fails the run. @@ -243,7 +248,7 @@ jobs: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 DANGER_CI_ON_HOST_FOLDERS: 1 FILE_ENV: ${{ matrix.file-env }} - # Read by the steps macos-arm64-unit shares, so it is set per job. + # Read by the steps the macOS jobs share, so it is set per job. MATRIX_NAME: ${{ matrix.name }} TEST_RUNNER_FAILFAST: 'false' MAKEJOBS: '-j4' @@ -315,7 +320,8 @@ jobs: with: cache-provider: gha - - name: CI script + - &CI_SCRIPT + name: CI script run: ./ci/test_run_all.sh - name: Report unit-test evidence @@ -347,9 +353,10 @@ jobs: - name: Save caches uses: ./.github/actions/save-caches - # Apple Silicon runs natively on the host, without Docker, so it is a job of - # its own rather than a matrix entry of unit. It shares unit's source checks - # and evidence report. + # macOS runs natively on the host, without Docker, so each architecture is a + # job of its own rather than a matrix entry of unit. Both share unit's source + # checks, CI script and evidence report, and build with WITH_MLKEM_NATIVE=ON: + # these are the only jobs that run ML-KEM's native code on macOS. macos-arm64-unit: name: macOS arm64 unit tests needs: resolve-source @@ -365,41 +372,97 @@ jobs: MAKEJOBS: '-j3' TEST_RUNNER_JOBS: '3' CTEST_JOBS: '3' + # Read by the steps the macOS jobs share: the runner's uname -m, the + # ML-KEM backend (arithmetic/Keccak) that must run, and the evidence line. + EXPECTED_MACHINE: arm64 + MLKEM_EXPECTED_BACKEND: aarch64-neon/aarch64 + PHASE: mlkem_tests, bip324_tests and net_tests natively on Apple Silicon, where ML-KEM runs its native AArch64 code. steps: - *REQUIRE_SOURCE - *CHECKOUT_SOURCE - *VERIFY_SOURCE - - name: Require Apple Silicon + - &REQUIRE_MACHINE + name: Require runner architecture shell: bash run: | set -euo pipefail machine="$(uname -m)" - if [ "$machine" != "arm64" ]; then - echo "::error::macos-arm64-unit must run on arm64, but uname -m reports '$machine'" >&2 + if [ "$machine" != "$EXPECTED_MACHINE" ]; then + echo "::error::${GITHUB_JOB} must run on ${EXPECTED_MACHINE}, but uname -m reports '$machine'" >&2 exit 1 fi echo "uname -m: $machine" # The runner context is not available in job-level env, so the build - # directory, which the report step reads, is set here. - - name: Set build directory + # directory, which the later steps read, is set here. + - &SET_BUILD_DIR + name: Set build directory shell: bash run: echo "BASE_BUILD_DIR=${RUNNER_TEMP}/build" >> "$GITHUB_ENV" - - name: Install Homebrew packages + - &INSTALL_HOMEBREW_PACKAGES + name: Install Homebrew packages env: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 run: brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent - - name: CI script - run: ./ci/test_run_all.sh + - *CI_SCRIPT - - name: Report unit-test evidence + # The suites passed, but on which code? A runner whose CPU lacks an + # extension the native code needs would pass them on portable C, so + # require the native backend by name. mlkem_tests/backend_selection logs + # the compiled and the active backend. + - &REQUIRE_NATIVE_MLKEM + name: Require native ML-KEM + shell: bash + run: | + set -euo pipefail + expected="compiled ${MLKEM_EXPECTED_BACKEND}, active ${MLKEM_EXPECTED_BACKEND}" + log="$("${BASE_BUILD_DIR}/bin/test_qbit" --run_test=mlkem_tests/backend_selection --log_level=message --color_output=no 2>&1)" \ + || { printf '%s\n' "$log"; echo "::error::mlkem_tests/backend_selection failed" >&2; exit 1; } + printf '%s\n' "$log" + if ! printf '%s\n' "$log" | grep -qxF "$expected"; then + echo "::error::${GITHUB_JOB} must run native ML-KEM, but the test log lacks '${expected}'" >&2 + sysctl machdep.cpu.brand_string machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null || true + exit 1 + fi + + - &MACOS_REPORT_EVIDENCE + name: Report unit-test evidence if: ${{ always() }} shell: bash env: RESOLVED_SHA: ${{ needs.resolve-source.outputs.resolved_sha }} REQUESTED_REF: ${{ needs.resolve-source.outputs.requested_ref }} - PHASE: mlkem_tests, bip324_tests and net_tests natively on Apple Silicon, where ML-KEM runs its native AArch64 code. run: *REPORT_EVIDENCE + + macos-x86_64-unit: + name: macOS x86_64 unit tests + needs: resolve-source + if: ${{ needs.resolve-source.outputs.run_macos_x86_64_unit == 'true' }} + # GitHub's Intel macOS image (4 CPUs); macos-15 itself is Apple Silicon. + runs-on: macos-15-intel + timeout-minutes: 120 + env: + DANGER_RUN_CI_ON_HOST: 1 + FILE_ENV: ./ci/test/00_setup_env_mac_native_pq.sh + MATRIX_NAME: macOS x86_64 unit tests + BASE_ROOT_DIR: ${{ github.workspace }} + TEST_RUNNER_FAILFAST: 'false' + MAKEJOBS: '-j4' + TEST_RUNNER_JOBS: '4' + CTEST_JOBS: '4' + EXPECTED_MACHINE: x86_64 + MLKEM_EXPECTED_BACKEND: x86_64-avx2/x86_64-avx2 + PHASE: mlkem_tests, bip324_tests and net_tests natively on an Intel Mac, where ML-KEM runs its native x86_64 code. + steps: + - *REQUIRE_SOURCE + - *CHECKOUT_SOURCE + - *VERIFY_SOURCE + - *REQUIRE_MACHINE + - *SET_BUILD_DIR + - *INSTALL_HOMEBREW_PACKAGES + - *CI_SCRIPT + - *REQUIRE_NATIVE_MLKEM + - *MACOS_REPORT_EVIDENCE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 651ee0dd49..0f439db648 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: profile: ${{ steps.classify.outputs.profile }} release_policy_only: ${{ steps.classify.outputs.release_policy_only }} source_validation_required: ${{ steps.classify.outputs.source_validation_required }} - pq_aarch64_required: ${{ steps.classify.outputs.pq_aarch64_required }} + pq_unit_required: ${{ steps.classify.outputs.pq_unit_required }} steps: - name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 @@ -1073,16 +1073,18 @@ jobs: build/doc/rpc/** # Changes to the ML-KEM library, the v2 transport, or their build and CI - # wiring (ci/checks/classify_merge_profile.py) must pass the aarch64 job of - # the PQ workflow on the change itself: for a pull request, its merge commit. - pq-aarch64: - name: 'PQ aarch64 unit tests' + # wiring (ci/checks/classify_merge_profile.py) must pass these jobs of the PQ + # workflow on the change itself (for a pull request, its merge commit), on + # GitHub-hosted runners: AArch64 Linux, and macOS on Apple Silicon and on + # Intel, the only jobs that build and run ML-KEM's native code on macOS. + pq-unit: + name: 'PQ unit tests' needs: classify-changes - if: ${{ needs.classify-changes.outputs.pq_aarch64_required == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} + if: ${{ needs.classify-changes.outputs.pq_unit_required == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} uses: ./.github/workflows/ci-pq.yml with: source_ref: ${{ github.sha }} - jobs: aarch64-unit + jobs: aarch64-unit,macos-arm64-unit,macos-x86_64-unit full-validation-gate: name: 'Full Validation Gate' @@ -1097,7 +1099,7 @@ jobs: - lint - public-docs-lint - rpc-docs - - pq-aarch64 + - pq-unit if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} steps: - name: Require trusted label for fork PRs @@ -1120,8 +1122,8 @@ jobs: LINT_RESULT: ${{ needs.lint.result }} PUBLIC_DOCS_LINT_RESULT: ${{ needs.public-docs-lint.result }} RPC_DOCS_RESULT: ${{ needs.rpc-docs.result }} - PQ_AARCH64_REQUIRED: ${{ needs.classify-changes.outputs.pq_aarch64_required }} - PQ_AARCH64_RESULT: ${{ needs.pq-aarch64.result }} + PQ_UNIT_REQUIRED: ${{ needs.classify-changes.outputs.pq_unit_required }} + PQ_UNIT_RESULT: ${{ needs.pq-unit.result }} run: | set -euo pipefail @@ -1140,21 +1142,21 @@ jobs: check_result "classify validation profile" "${CLASSIFY_CHANGES_RESULT}" echo "CI profile: ${CI_PROFILE:-unavailable}" - # The aarch64 PQ job may be skipped only when the classifier says this - # change does not need it. - case "${PQ_AARCH64_REQUIRED}" in + # The PQ unit jobs (AArch64 Linux, macOS arm64 and x86_64) may be + # skipped only when the classifier says this change does not need them. + case "${PQ_UNIT_REQUIRED}" in true) - check_result "PQ aarch64 unit tests" "${PQ_AARCH64_RESULT}" + check_result "PQ unit tests" "${PQ_UNIT_RESULT}" ;; false) - echo "PQ aarch64 unit tests: ${PQ_AARCH64_RESULT} (not required for this change)" - if [[ "${PQ_AARCH64_RESULT}" != "skipped" && "${PQ_AARCH64_RESULT}" != "success" ]]; then - echo "::error title=Full Validation failed::PQ aarch64 unit tests result was ${PQ_AARCH64_RESULT}." + echo "PQ unit tests: ${PQ_UNIT_RESULT} (not required for this change)" + if [[ "${PQ_UNIT_RESULT}" != "skipped" && "${PQ_UNIT_RESULT}" != "success" ]]; then + echo "::error title=Full Validation failed::PQ unit tests result was ${PQ_UNIT_RESULT}." failed=1 fi ;; *) - echo "::error title=Full Validation failed::The classifier did not report whether the PQ aarch64 job is required (pq_aarch64_required='${PQ_AARCH64_REQUIRED}')." + echo "::error title=Full Validation failed::The classifier did not report whether the PQ unit jobs are required (pq_unit_required='${PQ_UNIT_REQUIRED}')." failed=1 ;; esac diff --git a/.github/workflows/required-merge-gate.yml b/.github/workflows/required-merge-gate.yml index 0c5f1b9b0f..b2b4bdff62 100644 --- a/.github/workflows/required-merge-gate.yml +++ b/.github/workflows/required-merge-gate.yml @@ -143,7 +143,7 @@ jobs: # Job selection, source resolution and unit-test evidence of the PQ and # nightly workflows, which run outside pull requests, the nightly - # workflow's scheduled source, the PQ canary tools, and the aarch64 PQ job + # workflow's scheduled source, the PQ canary tools, and the PQ unit jobs # that Full Validation requires for ML-KEM and transport changes. - name: Test PQ CI workflow contract run: | diff --git a/ci/checks/classify_merge_profile.py b/ci/checks/classify_merge_profile.py index bb3fa82fb7..f00e503cbc 100755 --- a/ci/checks/classify_merge_profile.py +++ b/ci/checks/classify_merge_profile.py @@ -56,10 +56,11 @@ "contrib/guix/repo-templates/qbit-guix.sigs/operator-keys/", ) # Changes to the ML-KEM library, the v2 transport, or their build and CI wiring -# must pass the aarch64 job of .github/workflows/ci-pq.yml before they merge. +# must pass the aarch64-unit, macos-arm64-unit and macos-x86_64-unit jobs of +# .github/workflows/ci-pq.yml before they merge (the pq-unit job of ci.yml). # ci/checks/test_pq_merge_gate_contract.py fails if a CMake file, workflow or # ci/ file that mentions ML-KEM is missing here. -PQ_AARCH64_FILES = frozenset( +PQ_UNIT_FILES = frozenset( { # The build: the WITH_MLKEM_NATIVE option, the library and its link # into bitcoin_node, the run-time CPU detection, the configure header, @@ -100,7 +101,7 @@ "src/test/net_tests.cpp", } ) -PQ_AARCH64_PREFIXES = ( +PQ_UNIT_PREFIXES = ( "src/bip324", "src/compat/cpu_features.", "src/crypto/mlkem", @@ -157,9 +158,9 @@ def github_metadata_only(self) -> bool: return self.profile == GITHUB_METADATA_PROFILE @property - def pq_aarch64_required(self) -> bool: - """Whether the aarch64 PQ job must pass. A path that cannot be read requires it.""" - return bool(self.invalid_paths) or any(is_pq_aarch64_path(path) for path in self.paths) + def pq_unit_required(self) -> bool: + """Whether the PQ unit jobs must pass. A path that cannot be read requires them.""" + return bool(self.invalid_paths) or any(is_pq_unit_path(path) for path in self.paths) def normalize_path(path: str) -> str | None: @@ -203,8 +204,8 @@ def is_github_metadata_path(path: str) -> bool: return path in GITHUB_METADATA_FILES or path.startswith(GITHUB_METADATA_PREFIXES) -def is_pq_aarch64_path(path: str) -> bool: - return path in PQ_AARCH64_FILES or path.startswith(PQ_AARCH64_PREFIXES) +def is_pq_unit_path(path: str) -> bool: + return path in PQ_UNIT_FILES or path.startswith(PQ_UNIT_PREFIXES) def classify_paths(paths: list[str] | tuple[str, ...]) -> Classification: @@ -283,7 +284,7 @@ def github_outputs(classification: Classification) -> dict[str, str]: "touched_rpc_docs": bool_output(bool(classification.rpc_docs_paths)), "touched_public_docs": bool_output(bool(classification.public_docs_paths)), "touched_github_metadata": bool_output(bool(classification.github_metadata_paths)), - "pq_aarch64_required": bool_output(classification.pq_aarch64_required), + "pq_unit_required": bool_output(classification.pq_unit_required), } @@ -297,7 +298,7 @@ def describe_classification(classification: Classification) -> str: lines = [ f"validation_profile={classification.profile}", f"changed_count={len(classification.paths)}", - f"pq_aarch64_required={bool_output(classification.pq_aarch64_required)}", + f"pq_unit_required={bool_output(classification.pq_unit_required)}", ] if classification.release_policy_paths: diff --git a/ci/checks/test_ci_pq_contract.py b/ci/checks/test_ci_pq_contract.py index 078febfc7d..e0fff0d788 100755 --- a/ci/checks/test_ci_pq_contract.py +++ b/ci/checks/test_ci_pq_contract.py @@ -56,6 +56,8 @@ } GATE_WORKFLOW = svc.WORKFLOW_DIR / "required-merge-gate.yml" TEST_SCRIPT = REPO_ROOT / "ci" / "test" / "03_test_script.sh" +MLKEM_TESTS = REPO_ROOT / "src" / "test" / "mlkem_tests.cpp" +MLKEM_BACKEND_HEADERS = [REPO_ROOT / "src" / "crypto" / name for name in ("mlkem_arith_backend.h", "mlkem_fips202_backend.h")] RESOLVER_JOB = svc.RESOLVER_JOB TRIGGER_STEP = "Check how the run started" REQUESTED_STEP = "Resolve requested source" @@ -66,8 +68,15 @@ AARCH64_JOB = "aarch64-unit" S390X_JOB = "s390x-unit" MACOS_JOB = "macos-arm64-unit" +MACOS_X86_64_JOB = "macos-x86_64-unit" +# Each macOS job: (runner label, uname -m, the ML-KEM backend that must run, its +# word in the evidence line). The backend is read from the headers that name it. +MACOS_JOBS = { + MACOS_JOB: ("macos-15", "arm64", "aarch64", "native AArch64"), + MACOS_X86_64_JOB: ("macos-15-intel", "x86_64", "x86_64", "native x86_64"), +} PQ_MATRIX_JOBS = [AARCH64_JOB, S390X_JOB] -ALL_PQ_JOBS = {PQ_UNIT_JOB: PQ_MATRIX_JOBS, MACOS_JOB: []} +ALL_PQ_JOBS = {PQ_UNIT_JOB: PQ_MATRIX_JOBS, MACOS_JOB: [], MACOS_X86_64_JOB: []} PQ_SUITES = ["mlkem_tests", "bip324_tests", "net_tests"] AARCH64_SUITES = PQ_SUITES NIGHTLY_MATRIX_JOBS = ["arm32-unit-1", "arm32-unit-2", "previous-releases", "fuzz"] @@ -157,6 +166,18 @@ def running_jobs(workflow: dict[str, Any], context: dict[str, Any]) -> dict[str, return running +def mlkem_backend_names() -> dict[str, str]: + """The arithmetic/Keccak backend names GetBackendNames() reports, per architecture, read from the backend headers.""" + names: dict[str, list[str]] = {} + for header in MLKEM_BACKEND_HEADERS: + text = header.read_text(encoding="utf8") + for arch, name in re.findall(r'^#(?:el)?if defined\(MLK_SYS_(X86_64|AARCH64)\)\n#define QBIT_MLKEM_\w+_BACKEND_NAME "([^"]+)"', + text, re.M): + names.setdefault(arch.lower(), []).append(name) + assert all(len(pair) == 2 for pair in names.values()) and set(names) == {"x86_64", "aarch64"}, names + return {arch: "/".join(pair) for arch, pair in names.items()} + + def source_env(env_file: str, names: list[str], extra_env: dict[str, str] | None = None) -> dict[str, str]: """Source a ci/test env file in bash and return the effective values of names.""" script = f"set -e\nsource {env_file}\n" + "".join(f'printf "%s=%s\\n" {name} "${{{name}:-}}"\n' for name in names) @@ -462,7 +483,11 @@ def test_named_dispatch_runs_only_selected_jobs(self) -> None: (PQ, {"jobs": AARCH64_JOB}, {PQ_UNIT_JOB: [AARCH64_JOB]}), (PQ, {"jobs": S390X_JOB}, {PQ_UNIT_JOB: [S390X_JOB]}), (PQ, {"jobs": MACOS_JOB}, {MACOS_JOB: []}), + (PQ, {"jobs": MACOS_X86_64_JOB}, {MACOS_X86_64_JOB: []}), (PQ, {"jobs": f" {MACOS_JOB} , {AARCH64_JOB} "}, {PQ_UNIT_JOB: [AARCH64_JOB], MACOS_JOB: []}), + # What Full Validation's pq-unit job passes. + (PQ, {"jobs": f"{AARCH64_JOB},{MACOS_JOB},{MACOS_X86_64_JOB}"}, + {PQ_UNIT_JOB: [AARCH64_JOB], MACOS_JOB: [], MACOS_X86_64_JOB: []}), (PQ, {"jobs": ""}, ALL_PQ_JOBS), (NIGHTLY, {"run_test_matrix": True, "run_scanners": False, "jobs": "previous-releases"}, {"nightly-matrix": ["previous-releases"]}), @@ -738,7 +763,7 @@ def group(context: dict[str, Any]) -> str: def test_rejects_unknown_repeated_and_empty_job_names(self) -> None: bad = { PQ: ["aarch64", "AARCH64-UNIT", f"{AARCH64_JOB},{AARCH64_JOB}", f"{AARCH64_JOB},", ",", "s390x", "macos-arm64", - f"{MACOS_JOB},{MACOS_JOB}"], + f"{MACOS_JOB},{MACOS_JOB}", "macos-x86_64", "macos-intel"], NIGHTLY: ["previous_releases", "previous-releases,previous-releases", "previous-releases,,fuzz", "unit"], } for key, values in bad.items(): @@ -793,11 +818,12 @@ def test_every_pq_job_checks_out_the_resolved_commit(self) -> None: workflow = self.workflows[PQ] context = self.resolve(PQ, self.dispatch(PQ, source_ref=svc.MAINTAINED_BRANCH, jobs=AARCH64_JOB)) refs = svc.check_source_wiring(workflow, context) - self.assertEqual(refs, {PQ_UNIT_JOB: fixture.maintained_sha, MACOS_JOB: fixture.maintained_sha}) - # The macOS job shares the unit job's guard, checkout and verification steps. - for step_name in (svc.GUARD_STEP, svc.CHECKOUT_STEP, svc.VERIFY_STEP): - self.assertEqual(svc.find_step(workflow["jobs"][MACOS_JOB], step_name), - svc.find_step(workflow["jobs"][PQ_UNIT_JOB], step_name), step_name) + self.assertEqual(refs, {job: fixture.maintained_sha for job in ALL_PQ_JOBS}) + # The macOS jobs share the unit job's guard, checkout and verification steps. + for macos_job in MACOS_JOBS: + for step_name in (svc.GUARD_STEP, svc.CHECKOUT_STEP, svc.VERIFY_STEP): + self.assertEqual(svc.find_step(workflow["jobs"][macos_job], step_name), + svc.find_step(workflow["jobs"][PQ_UNIT_JOB], step_name), (macos_job, step_name)) mutated = copy.deepcopy(workflow) checkout = svc.find_step(mutated["jobs"][PQ_UNIT_JOB], svc.CHECKOUT_STEP) @@ -884,22 +910,34 @@ def test_s390x_job_cross_compiles_and_emulates_only_the_tests(self) -> None: # ctest runs each test through the emulator, so test_qbit is never started directly. self.assertIn("-DCMAKE_CROSSCOMPILING_EMULATOR='qemu-s390x;-L;/usr/s390x-linux-gnu'", effective["BITCOIN_CONFIG"]) - def test_macos_job_runs_natively_on_apple_silicon(self) -> None: - fixture = self.fixture + def test_macos_jobs_run_natively_on_apple_silicon_and_intel(self) -> None: workflow = self.workflows[PQ] - job = workflow["jobs"][MACOS_JOB] - self.assertNotIn(MACOS_JOB, [entry["job"] for entry in catalog(workflow)]) - self.assertEqual(job["runs-on"], "macos-15") - context = self.resolve(PQ, self.dispatch(PQ, jobs=MACOS_JOB)) + catalog_names = [entry["job"] for entry in catalog(workflow)] + # Both jobs run the same steps; only their runner and job env differ. + self.assertEqual(workflow["jobs"][MACOS_JOB]["steps"], workflow["jobs"][MACOS_X86_64_JOB]["steps"]) + backends = mlkem_backend_names() + for job_id, (runner, machine, arch, evidence_word) in MACOS_JOBS.items(): + with self.subTest(job=job_id): + self.assertNotIn(job_id, catalog_names) + self.check_macos_job(workflow, job_id, runner, machine, backends[arch], evidence_word) + + def check_macos_job(self, workflow: dict[str, Any], job_id: str, runner: str, machine: str, backend: str, + evidence_word: str) -> None: + fixture = self.fixture + job = workflow["jobs"][job_id] + self.assertEqual(job["runs-on"], runner) + context = self.resolve(PQ, self.dispatch(PQ, jobs=job_id)) env = svc.job_env(workflow, job, context) self.assertEqual(env["DANGER_RUN_CI_ON_HOST"], "1", "no Docker on the macOS host") env_file = env["FILE_ENV"] self.assertTrue(os.access(REPO_ROOT / env_file, os.X_OK), env_file) self.assertEqual(env["MATRIX_NAME"], job["name"]) + self.assertEqual(env["EXPECTED_MACHINE"], machine) + self.assertEqual(env["MLKEM_EXPECTED_BACKEND"], backend) names = ["CI_OS_NAME", "NO_DEPENDS", "CI_BUILD_TARGET", "CTEST_REGEX", "CTEST_EXPECTED_SUITES", "RUN_FUNCTIONAL_TESTS", "BITCOIN_CONFIG"] effective = source_env(env_file, names) - # Losing the native AArch64 backend must fail the configure step, not fall back quietly. + # Losing the native backend must fail the configure step, not fall back quietly. self.assertIn("-DWITH_MLKEM_NATIVE=ON", effective["BITCOIN_CONFIG"].split()) self.assertEqual(effective["CI_OS_NAME"], "macos") self.assertEqual(effective["NO_DEPENDS"], "1") @@ -908,9 +946,10 @@ def test_macos_job_runs_natively_on_apple_silicon(self) -> None: self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") regex = re.compile(effective["CTEST_REGEX"]) self.assertEqual([name for name in SUITE_INVENTORY if regex.search(name)], sorted(PQ_SUITES)) + github_env = svc.github_env_for(context, job_id, workflow["name"]) - # The report step reads the evidence from BASE_BUILD_DIR, which a step - # sets from RUNNER_TEMP: the runner context is unavailable in job env. + # The report and native checks read the build from BASE_BUILD_DIR, which + # a step sets from RUNNER_TEMP: the runner context is unavailable in job env. self.assertNotIn("BASE_BUILD_DIR", job["env"]) set_dir = svc.find_step(job, "Set build directory") self.assertLess(svc.step_index(job, "Set build directory"), svc.step_index(job, "CI script")) @@ -919,25 +958,59 @@ def test_macos_job_runs_natively_on_apple_silicon(self) -> None: github_env_file.touch() step_env = svc.step_env(workflow, job, set_dir, context) step_env.update({"RUNNER_TEMP": "/runner/temp", "GITHUB_ENV": str(github_env_file)}) - result = svc.run_step(fixture, set_dir, step_env, fixture.root, svc.github_env_for(context, MACOS_JOB, workflow["name"])) + result = svc.run_step(fixture, set_dir, step_env, fixture.root, github_env) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(github_env_file.read_text(encoding="utf8"), "BASE_BUILD_DIR=/runner/temp/build\n") - # The architecture check refuses anything but arm64. - step = svc.find_step(job, "Require Apple Silicon") - self.assertLess(svc.step_index(job, "Require Apple Silicon"), svc.step_index(job, "CI script")) + # The architecture check refuses any other machine. + step = svc.find_step(job, "Require runner architecture") + self.assertLess(svc.step_index(job, "Require runner architecture"), svc.step_index(job, "CI script")) with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: stubs = Path(tmp) (stubs / "uname").write_text('#!/bin/sh\necho "$STUB_MACHINE"\n', encoding="utf8") (stubs / "uname").chmod(0o755) - for machine, ok in (("arm64", True), ("x86_64", False)): - with self.subTest(machine=machine): + for other in ("arm64", "x86_64"): + with self.subTest(machine=other): step_env = svc.step_env(workflow, job, step, context) - step_env.update({"PATH": f"{stubs}:{os.environ['PATH']}", "STUB_MACHINE": machine}) - result = svc.run_step(fixture, step, step_env, fixture.root, svc.github_env_for(context, MACOS_JOB, workflow["name"])) - self.assertEqual(result.returncode == 0, ok, result.stderr) - if not ok: - self.assertIn("must run on arm64", result.stderr) + step_env.update({"PATH": f"{stubs}:{os.environ['PATH']}", "STUB_MACHINE": other}) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode == 0, other == machine, result.stderr) + if other != machine: + self.assertIn(f"{job_id} must run on {machine}", result.stderr) + + # After the suites pass, the native backend must be the one that ran. + step = svc.find_step(job, "Require native ML-KEM") + self.assertLess(svc.step_index(job, "CI script"), svc.step_index(job, "Require native ML-KEM")) + self.assertLess(svc.step_index(job, "Require native ML-KEM"), svc.step_index(job, REPORT_STEP)) + native_line = f"compiled {backend}, active {backend}" + cases = [ + ("native", native_line, 0, True), + ("portable on a CPU without an extension", f"compiled {backend}, active portable/portable", 0, False), + ("portable build", "compiled portable/portable, active portable/portable", 0, False), + ("other native backend", "compiled other/other, active other/other", 0, False), + ("only part of the line", f"compiled {backend}, active {backend}x", 0, False), + ("test failed", native_line, 201, False), + ] + for label, line, exit_code, ok in cases: + with self.subTest(job=job_id, native_check=label), tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + build = Path(tmp) + (build / "bin").mkdir() + stub = build / "bin" / "test_qbit" + stub.write_text("#!/bin/sh\n" + 'printf "%s\\n" "$*" > "$(dirname "$0")/args"\n' + f"echo 'Running 1 test case...'\necho '{line}'\nexit {exit_code}\n", encoding="utf8") + stub.chmod(0o755) + step_env = svc.step_env(workflow, job, step, context) + step_env["BASE_BUILD_DIR"] = str(build) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode == 0, ok, result.completed.stdout + result.stderr) + self.assertEqual((build / "bin" / "args").read_text(encoding="utf8").split(), + ["--run_test=mlkem_tests/backend_selection", "--log_level=message", "--color_output=no"]) + if not ok: + self.assertIn("::error::", result.stderr) + # The line the step looks for is the one backend_selection logs. + self.assertIn('BOOST_TEST_MESSAGE("compiled " << compiled_arith << "/" << compiled_keccak << ", active " << ' + 'active.arith << "/" << active.keccak);', MLKEM_TESTS.read_text(encoding="utf8")) # Evidence is reported exactly as for the matrix jobs. report = svc.find_step(job, REPORT_STEP) @@ -949,10 +1022,10 @@ def test_macos_job_runs_natively_on_apple_silicon(self) -> None: (build / "ctest-evidence" / "summary.md").write_text("| mlkem_tests | passed | 0.5 |\n", encoding="utf8") step_env = svc.step_env(workflow, job, report, context) step_env["BASE_BUILD_DIR"] = str(build) - result = svc.run_step(fixture, report, step_env, fixture.root, svc.github_env_for(context, MACOS_JOB, workflow["name"])) + result = svc.run_step(fixture, report, step_env, fixture.root, github_env) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("## macOS arm64 unit tests", result.summary) - self.assertIn("native AArch64", result.summary) + self.assertIn(f"## {job['name']}", result.summary) + self.assertIn(evidence_word, result.summary) self.assertIn("| mlkem_tests | passed | 0.5 |", result.summary) def test_contexts_are_available_where_used(self) -> None: diff --git a/ci/checks/test_classify_merge_profile.py b/ci/checks/test_classify_merge_profile.py index 16521b8221..917059be4f 100755 --- a/ci/checks/test_classify_merge_profile.py +++ b/ci/checks/test_classify_merge_profile.py @@ -242,7 +242,7 @@ def test_github_outputs_report_github_metadata_profile(self) -> None: self.assertEqual(outputs["source_validation_required"], "false") self.assertEqual(outputs["touched_github_metadata"], "true") - def test_pq_aarch64_job_required_for_mlkem_and_transport_paths(self) -> None: + def test_pq_unit_jobs_required_for_mlkem_and_transport_paths(self) -> None: required = [ "src/mlkem-native/mlkem/src/kem.c", "src/crypto/mlkem.cpp", @@ -289,19 +289,19 @@ def test_pq_aarch64_job_required_for_mlkem_and_transport_paths(self) -> None: for path in required: with self.subTest(path=path): classification = self.classify([path]) - self.assertTrue(classification.pq_aarch64_required) - self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_aarch64_required"], "true") + self.assertTrue(classification.pq_unit_required) + self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_unit_required"], "true") self.assertEqual(classification.profile, classify_merge_profile.SOURCE_PROFILE) # One matching path among others is enough. - self.assertTrue(self.classify(["doc/user/README.md", "src/bip324.cpp"]).pq_aarch64_required) + self.assertTrue(self.classify(["doc/user/README.md", "src/bip324.cpp"]).pq_unit_required) - def test_pq_aarch64_job_not_required_elsewhere(self) -> None: + def test_pq_unit_jobs_not_required_elsewhere(self) -> None: for paths in ( ["src/wallet/wallet.cpp"], ["src/netbase.cpp", "src/net_processing.cpp", "src/compat/compat.h"], ["src/test/net_peer_eviction_tests.cpp"], ["ci/test/00_setup_env_native_asan.sh", ".github/workflows/rpc-perf-manual.yml"], - # Wiring shared by every CI job, and ML-KEM tooling the aarch64 job never runs. + # Wiring shared by every CI job, and ML-KEM tooling the PQ unit jobs never run. ["ci/test/01_base_install.sh", "ci/test/02_run_container.py", ".github/actions/configure-docker/action.yml"], ["src/crypto/CMakeLists.txt", "src/test/cpu_features_tests.cpp"], ["test/lint/mlkem-native-check.sh", "contrib/devtools/mlkem-native.pin", "contrib/devtools/update-mlkem-native.sh"], @@ -311,12 +311,12 @@ def test_pq_aarch64_job_not_required_elsewhere(self) -> None: ): with self.subTest(paths=paths): classification = self.classify(paths) - self.assertFalse(classification.pq_aarch64_required) - self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_aarch64_required"], "false") + self.assertFalse(classification.pq_unit_required) + self.assertEqual(classify_merge_profile.github_outputs(classification)["pq_unit_required"], "false") - def test_pq_aarch64_job_required_for_unreadable_paths(self) -> None: - self.assertTrue(self.classify(["../src/bip324.cpp"]).pq_aarch64_required) - self.assertTrue(self.classify(["/abs/path"]).pq_aarch64_required) + def test_pq_unit_jobs_required_for_unreadable_paths(self) -> None: + self.assertTrue(self.classify(["../src/bip324.cpp"]).pq_unit_required) + self.assertTrue(self.classify(["/abs/path"]).pq_unit_required) def test_require_release_policy_only_cli_rejects_outside_path(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/ci/checks/test_pq_merge_gate_contract.py b/ci/checks/test_pq_merge_gate_contract.py index 177a842a49..0895c9d159 100755 --- a/ci/checks/test_pq_merge_gate_contract.py +++ b/ci/checks/test_pq_merge_gate_contract.py @@ -2,18 +2,20 @@ # Copyright (c) 2026-present The qbit core developers # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. -"""Contract for the aarch64 PQ job in Full Validation (#184 part 3, work item C3). +"""Contract for the PQ unit jobs in Full Validation (#184 part 3, work item C3). Pull requests that touch the ML-KEM library, the v2 transport, or their build -and CI wiring must pass the aarch64-unit job of ci-pq.yml before they merge; -others skip it. These tests run the real pieces of that chain: +and CI wiring must pass the aarch64-unit, macos-arm64-unit and +macos-x86_64-unit jobs of ci-pq.yml before they merge; others skip all three. +These tests run the real pieces of that chain: * ci/checks/classify_merge_profile.py, as the classify-changes job runs it, - reports pq_aarch64_required; -* the pq-aarch64 job of ci.yml calls ci-pq.yml with jobs aarch64-unit on the + reports pq_unit_required; +* the pq-unit job of ci.yml calls ci-pq.yml with those three jobs on the change's own commit (a pull request's merge commit) exactly when that output - is true, and ci-pq.yml's selection step then selects only aarch64-unit; -* full-validation-gate's shell accepts a skipped job only when the classifier + is true, and ci-pq.yml then runs exactly those three, so one output and one + call make each of them required exactly when the others are; +* full-validation-gate's shell accepts a skipped call only when the classifier says it is not required, and fails on a missing classifier output; * the changed-file lists that feed the classifier name both sides of a rename; * every CMake file, workflow or ci/ file that mentions ML-KEM classifies as @@ -42,11 +44,15 @@ GATE_WORKFLOW = svc.WORKFLOW_DIR / "required-merge-gate.yml" CLASSIFIER = REPO_ROOT / "ci" / "checks" / "classify_merge_profile.py" CLASSIFY_JOB = "classify-changes" -PQ_JOB = "pq-aarch64" +PQ_JOB = "pq-unit" GATE_JOB = "full-validation-gate" GATE_STEP = "Check validation results" AARCH64_JOB = "aarch64-unit" -OUTPUT = "pq_aarch64_required" +# The ci-pq.yml jobs Full Validation requires: matrix entries of its unit job, +# and jobs of their own (the macOS ones), by the name the jobs input uses. +REQUIRED_PQ_JOBS = [AARCH64_JOB, "macos-arm64-unit", "macos-x86_64-unit"] +PQ_UNIT_MATRIX_JOB = "unit" +OUTPUT = "pq_unit_required" MERGE_SHA = "1234567890abcdef1234567890abcdef12345678" HEAD_SHA = "fedcba0987654321fedcba0987654321fedcba09" CHANGED_FILES_STEP = "Write changed file list" @@ -148,18 +154,32 @@ def test_classify_changes_exposes_the_output(self) -> None: self.assertIn("ci/checks/classify_merge_profile.py", classify_step["run"]) self.assertIn('--github-output "$GITHUB_OUTPUT"', classify_step["run"]) - def test_pq_job_calls_only_aarch64_on_the_change_itself(self) -> None: + def running_pq_jobs(self, outputs: dict[str, str]) -> dict[str, list[str]]: + """The ci-pq.yml jobs whose if passes with these selection outputs, with their matrix entries.""" + context = {"needs": {svc.RESOLVER_JOB: {"outputs": outputs, "result": "success"}}} + running: dict[str, list[str]] = {} + for job_id, job in self.pq["jobs"].items(): + if job_id == svc.RESOLVER_JOB or not svc._truthy(svc.render(job["if"], context)): + continue + matrix = (job.get("strategy") or {}).get("matrix") + running[job_id] = [entry["job"] for entry in json.loads(outputs["matrix"])["include"]] if matrix else [] + return running + + def test_pq_job_calls_the_required_jobs_on_the_change_itself(self) -> None: job = self.ci["jobs"][PQ_JOB] self.assertEqual(job["uses"], "./.github/workflows/ci-pq.yml") self.assertEqual(job["needs"], CLASSIFY_JOB) context = pull_request_context({OUTPUT: "true"}) rendered = {key: svc._to_text(svc.render(value, context)) for key, value in job["with"].items()} - self.assertEqual(rendered, {"source_ref": MERGE_SHA, "jobs": AARCH64_JOB}) + self.assertEqual(rendered, {"source_ref": MERGE_SHA, "jobs": ",".join(REQUIRED_PQ_JOBS)}) self.assertEqual(set(job["with"]), set(self.pq["on"]["workflow_call"]["inputs"])) outputs = self.select_pq_jobs(rendered["jobs"]) - self.assertEqual([entry["job"] for entry in json.loads(outputs["matrix"])["include"]], [AARCH64_JOB]) - self.assertEqual(outputs["run_macos_arm64_unit"], "false") + self.assertEqual(json.loads(outputs["selected"]), REQUIRED_PQ_JOBS) + # Exactly these run: aarch64-unit as the unit job's only entry, each + # macOS job as itself, and nothing else (s390x-unit stays on demand). + self.assertEqual(self.running_pq_jobs(outputs), + {PQ_UNIT_MATRIX_JOB: [AARCH64_JOB], "macos-arm64-unit": [], "macos-x86_64-unit": []}) def test_required_merge_gate_runs_this_contract(self) -> None: gate = svc.load_workflow(GATE_WORKFLOW) @@ -212,8 +232,8 @@ def test_gate_reads_the_job_and_the_classifier(self) -> None: self.assertIn(PQ_JOB, gate["needs"]) self.assertIn("always()", gate["if"]) env = svc.find_step(gate, GATE_STEP)["env"] - self.assertEqual(env["PQ_AARCH64_REQUIRED"], "${{ needs." + CLASSIFY_JOB + ".outputs." + OUTPUT + " }}") - self.assertEqual(env["PQ_AARCH64_RESULT"], "${{ needs." + PQ_JOB + ".result }}") + self.assertEqual(env["PQ_UNIT_REQUIRED"], "${{ needs." + CLASSIFY_JOB + ".outputs." + OUTPUT + " }}") + self.assertEqual(env["PQ_UNIT_RESULT"], "${{ needs." + PQ_JOB + ".result }}") def test_job_condition(self) -> None: self.assertTrue(self.job_runs(pull_request_context({OUTPUT: "true"}))) @@ -229,23 +249,23 @@ def test_job_condition(self) -> None: def test_gate_outcomes(self) -> None: cases = [ ("required and passed", "true", "success", True, ""), - ("required but skipped", "true", "skipped", False, "PQ aarch64 unit tests result was skipped"), - ("required but failed", "true", "failure", False, "PQ aarch64 unit tests result was failure"), - ("required but cancelled", "true", "cancelled", False, "PQ aarch64 unit tests result was cancelled"), + ("required but skipped", "true", "skipped", False, "PQ unit tests result was skipped"), + ("required but failed", "true", "failure", False, "PQ unit tests result was failure"), + ("required but cancelled", "true", "cancelled", False, "PQ unit tests result was cancelled"), ("not required, skipped", "false", "skipped", True, ""), - ("not required, failed anyway", "false", "failure", False, "PQ aarch64 unit tests result was failure"), - ("classifier output missing", "", "skipped", False, "did not report whether the PQ aarch64 job is required"), - ("classifier output garbled", "yes", "success", False, "did not report whether the PQ aarch64 job is required"), + ("not required, failed anyway", "false", "failure", False, "PQ unit tests result was failure"), + ("classifier output missing", "", "skipped", False, "did not report whether the PQ unit jobs are required"), + ("classifier output garbled", "yes", "success", False, "did not report whether the PQ unit jobs are required"), ] for label, required, result, ok, message in cases: with self.subTest(label): - run = self.run_gate(PQ_AARCH64_REQUIRED=required, PQ_AARCH64_RESULT=result) + run = self.run_gate(PQ_UNIT_REQUIRED=required, PQ_UNIT_RESULT=result) self.assertEqual(run.returncode == 0, ok, run.stdout + run.stderr) if message: self.assertIn(message, run.stdout) # A missing output fails even a change that needs no source validation. - run = self.run_gate(PQ_AARCH64_REQUIRED="", PQ_AARCH64_RESULT="skipped", + run = self.run_gate(PQ_UNIT_REQUIRED="", PQ_UNIT_RESULT="skipped", SOURCE_VALIDATION_REQUIRED="false", VALIDATION_PROFILE="public-docs") self.assertNotEqual(run.returncode, 0) @@ -257,7 +277,7 @@ def gate_for_pull_request(self, paths: list[str]) -> tuple[bool, subprocess.Comp runs = self.job_runs(pull_request_context(outputs)) source = outputs["source_validation_required"] others = OTHER_GATE_RESULTS if source == "true" else {key: "skipped" for key in OTHER_GATE_RESULTS} - gate = self.run_gate(PQ_AARCH64_REQUIRED=outputs[OUTPUT], PQ_AARCH64_RESULT="success" if runs else "skipped", + gate = self.run_gate(PQ_UNIT_REQUIRED=outputs[OUTPUT], PQ_UNIT_RESULT="success" if runs else "skipped", SOURCE_VALIDATION_REQUIRED=source, VALIDATION_PROFILE=outputs["profile"], **others) return runs, gate @@ -265,10 +285,10 @@ def test_matching_pull_request_requires_and_runs_the_job(self) -> None: runs, gate = self.gate_for_pull_request(MATCHING_PR) self.assertTrue(runs) self.assertEqual(gate.returncode, 0, gate.stdout + gate.stderr) - self.assertIn("PQ aarch64 unit tests: success", gate.stdout) + self.assertIn("PQ unit tests: success", gate.stdout) # Skipping it would fail the gate. outputs = classify(MATCHING_PR) - self.assertNotEqual(self.run_gate(PQ_AARCH64_REQUIRED=outputs[OUTPUT], PQ_AARCH64_RESULT="skipped").returncode, 0) + self.assertNotEqual(self.run_gate(PQ_UNIT_REQUIRED=outputs[OUTPUT], PQ_UNIT_RESULT="skipped").returncode, 0) def test_other_pull_requests_skip_the_job_and_pass(self) -> None: for paths in (DOCS_ONLY_PR, OTHER_SOURCE_PR): @@ -298,9 +318,9 @@ def test_wiring_that_mentions_mlkem_requires_the_job(self) -> None: self.assertIn(known, wiring) missing = [path for path in wiring if classify([path])[OUTPUT] != "true"] self.assertEqual(missing, [], - "FAIL: these files mention ML-KEM but do not require the aarch64 PQ job\n" - "Cause: a change to only one of them would skip pq-aarch64, and Full Validation accepts the skip\n" - "Fix: add them to PQ_AARCH64_FILES in ci/checks/classify_merge_profile.py") + "FAIL: these files mention ML-KEM but do not require the PQ unit jobs\n" + "Cause: a change to only one of them would skip pq-unit, and Full Validation accepts the skip\n" + "Fix: add them to PQ_UNIT_FILES in ci/checks/classify_merge_profile.py") if __name__ == "__main__": diff --git a/ci/test/00_setup_env_mac_native_pq.sh b/ci/test/00_setup_env_mac_native_pq.sh index 8de8148b3e..39b6601fa5 100755 --- a/ci/test/00_setup_env_mac_native_pq.sh +++ b/ci/test/00_setup_env_mac_native_pq.sh @@ -6,9 +6,10 @@ export LC_ALL=C.UTF-8 -# The macos-arm64-unit job of .github/workflows/ci-pq.yml: build only the -# unit-test binary natively on an Apple Silicon runner, without Docker, and run -# the post-quantum suites. This is the target that runs native AArch64 ML-KEM. +# The macos-arm64-unit and macos-x86_64-unit jobs of .github/workflows/ci-pq.yml: +# build only the unit-test binary natively on an Apple Silicon or Intel runner, +# without Docker, and run the post-quantum suites. These are the targets that +# run ML-KEM's native AArch64 and x86_64 code on macOS. export CONTAINER_NAME="ci_mac_native_pq" # macos does not use a container, but the env var is needed for logging export CMAKE_GENERATOR="Ninja" export CI_OS_NAME="macos" @@ -21,6 +22,6 @@ export CI_BUILD_TARGET=test_bitcoin # selected, executed and passed. CTEST_REGEX is derived from the same list. export CTEST_EXPECTED_SUITES="mlkem_tests bip324_tests net_tests" export CTEST_REGEX="^(${CTEST_EXPECTED_SUITES// /|})\$" -# WITH_MLKEM_NATIVE=ON makes losing the native AArch64 backend a configure error -# instead of a quiet fall back to portable C. +# WITH_MLKEM_NATIVE=ON makes losing the native backend a configure error instead +# of a quiet fall back to portable C. export BITCOIN_CONFIG="-DBUILD_GUI=OFF -DENABLE_WALLET=OFF -DENABLE_IPC=OFF -DWITH_ZMQ=OFF -DREDUCE_EXPORTS=ON -DWITH_MLKEM_NATIVE=ON" From c945ce5f7b01e4d1d198f68b29c5f46804f21f5c Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 12:43:08 -0400 Subject: [PATCH 12/20] test: check the portable override on AArch64 and per entry point The build-policy test counted assembly calls only for the x86_64 backend, so nothing in CI showed that the portable override reaches every AArch64 assembly routine. With upstream's x1 Keccak glue in place of qbit's mirror, 221 assembly calls ran with portable C forced, and every test passed: outputs agree, and the backend name comes from the hook. The counting harness now also runs on AArch64: a static binary built with aarch64-linux-gnu-g++, from a copy of mlkem_config.h without the AArch64 ELF refusal, and run under qemu-aarch64. A negative control builds upstream's x1 glue and requires the leaked calls to show. Each entry point also runs alone on a new thread while portable C is forced, and must make no assembly call: one that skipped BeginOperation() would run on the new thread's stale copy of the override. The earlier round trip could not show this, since KeyGen set the copy for the whole thread. The build smoke job installs g++-aarch64-linux-gnu and qemu-user and lists them as required, so the AArch64 case cannot skip. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/core-checks.yml | 26 +-- ci/checks/test_mlkem_build_policy.py | 263 +++++++++++++++++++++------ doc/subtrees/mlkem-native.md | 16 +- 3 files changed, 233 insertions(+), 72 deletions(-) diff --git a/.github/workflows/core-checks.yml b/.github/workflows/core-checks.yml index 9636846961..f6badd35df 100644 --- a/.github/workflows/core-checks.yml +++ b/.github/workflows/core-checks.yml @@ -165,18 +165,22 @@ jobs: - name: Test integrated libbitcoinpqc build policy run: python3 ci/checks/test_libbitcoinpqc_build_policy.py - # Every compiler the build-policy test uses, each listed as required so - # that none of its cases can skip: the AArch64 cross compiler proves that - # AArch64 ELF builds refuse native ML-KEM (no BTI landing pads), MinGW and - # Clang cover the Windows and Clang builds, and cc -m32 an x86 target - # without a backend. gcc-aarch64-linux-gnu only recommends its libc - # headers, so they are installed explicitly. For -m32, libc6-dev-i386 - # provides the 32-bit headers: gcc-multilib conflicts with the AArch64 - # cross compiler on Ubuntu 24.04. clang-21 comes from apt.llvm.org, as in - # ci/test/01_base_install.sh. + # Every compiler and tool the build-policy test uses, each listed as + # required so that none of its cases can skip: the AArch64 cross compiler + # proves that AArch64 ELF builds refuse native ML-KEM (no BTI landing + # pads), MinGW and Clang cover the Windows and Clang builds, and cc -m32 + # an x86 target without a backend. The AArch64 C++ cross compiler, its nm + # and qemu-aarch64 run the AArch64 backend's portable-override check: a + # static test binary under emulation. gcc-aarch64-linux-gnu only + # recommends its libc headers, so they are installed explicitly. For -m32, + # libc6-dev-i386 provides the 32-bit headers: gcc-multilib conflicts with + # the AArch64 cross compiler on Ubuntu 24.04. clang-21 comes from + # apt.llvm.org, as in ci/test/01_base_install.sh. - name: Test integrated mlkem-native build policy env: - MLKEM_POLICY_REQUIRED_COMPILERS: cc cc-m32 clang-21 aarch64-linux-gnu-gcc x86_64-w64-mingw32-gcc-posix + MLKEM_POLICY_REQUIRED_COMPILERS: >- + cc cc-m32 clang-21 aarch64-linux-gnu-gcc aarch64-linux-gnu-g++ aarch64-linux-gnu-nm qemu-aarch64 + x86_64-w64-mingw32-gcc-posix APT_LLVM_V: 21 run: | sudo install -d -m 0755 /etc/apt/keyrings @@ -187,7 +191,7 @@ jobs: | sudo tee "/etc/apt/sources.list.d/llvm-toolchain-${codename}-${APT_LLVM_V}.list" > /dev/null sudo apt-get update sudo apt-get install -y --no-install-recommends \ - gcc-aarch64-linux-gnu libc6-dev-arm64-cross \ + gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user \ gcc-mingw-w64-x86-64-posix \ libc6-dev-i386 \ "clang-${APT_LLVM_V}" "llvm-${APT_LLVM_V}" diff --git a/ci/checks/test_mlkem_build_policy.py b/ci/checks/test_mlkem_build_policy.py index 28aac4bfc3..807afb1e34 100755 --- a/ci/checks/test_mlkem_build_policy.py +++ b/ci/checks/test_mlkem_build_policy.py @@ -19,9 +19,11 @@ it installs: then a missing one fails. "cc-m32" stands for cc -m32 with 32-bit headers. Failures print the signature, cause and fix. -A fourth layer runs on x86_64 hosts: it links qbit's glue and wrapper with -every assembly routine wrapped by a call counter, and checks that the portable -override reaches each of them. +A fourth layer links qbit's glue and wrapper with every assembly routine +wrapped by a call counter, and checks that the portable override reaches each +of them: the x86_64 backend on x86_64 hosts, the AArch64 backend under +qemu-aarch64 (listed as qemu-aarch64 and aarch64-linux-gnu-g++ in +MLKEM_POLICY_REQUIRED_COMPILERS when required). """ from __future__ import annotations @@ -499,104 +501,249 @@ def test_guards(self) -> None: COVERAGE_HARNESS = r""" #include +#include #include +#include +#include #include unsigned long AsmCalls(); using namespace mlkem; -static std::vector RoundTrip() +static std::array Seed() { std::array seed{}; for (size_t i = 0; i < seed.size(); ++i) seed[i] = uint8_t(i * 7 + 1); - const std::array coins{}; + return seed; +} +static const std::array COINS{}; +static std::vector RoundTrip() +{ PublicKey ek; DecapsulationKey dk; Ciphertext ct; SharedSecret sent, received; - if (KeyGen(seed, ek, dk) != Error::NONE || CheckPublicKey(ek) != Error::NONE || - Encaps(ek, coins, ct, sent) != Error::NONE || Decaps(dk, ct, received) != Error::NONE) return {}; + if (KeyGen(Seed(), ek, dk) != Error::NONE || CheckPublicKey(ek) != Error::NONE || + Encaps(ek, COINS, ct, sent) != Error::NONE || Decaps(dk, ct, received) != Error::NONE) return {}; std::vector out(ek.begin(), ek.end()); out.insert(out.end(), ct.begin(), ct.end()); out.insert(out.end(), received.Bytes().begin(), received.Bytes().end()); return out; } +// The assembly calls of each entry point, run alone on a new thread, whose +// copy of the override is unset until the entry point takes it. +static std::array EachOperationOnAFreshThread(const PublicKey& ek, const DecapsulationKey& dk, const Ciphertext& ct) +{ + const std::function operations[4]{ + [] { PublicKey e; DecapsulationKey d; (void)KeyGen(Seed(), e, d); }, + [&] { (void)CheckPublicKey(ek); }, + [&] { Ciphertext c; SharedSecret s; (void)Encaps(ek, COINS, c, s); }, + [&] { SharedSecret s; (void)Decaps(dk, ct, s); }, + }; + std::array calls{}; + for (size_t i = 0; i < calls.size(); ++i) { + AsmCalls(); + std::thread{operations[i]}.join(); + calls[i] = AsmCalls(); + } + return calls; +} int main() { const auto detected{RoundTrip()}; const unsigned long detected_calls{AsmCalls()}; const auto active{GetBackendNames().arith}; + PublicKey ek; DecapsulationKey dk; Ciphertext ct; SharedSecret ss; + const bool inputs{KeyGen(Seed(), ek, dk) == Error::NONE && Encaps(ek, COINS, ct, ss) == Error::NONE}; + const auto fresh_detected{EachOperationOnAFreshThread(ek, dk, ct)}; std::vector portable; unsigned long portable_calls; + std::array fresh_portable; { ForcePortableForTesting force; + AsmCalls(); portable = RoundTrip(); portable_calls = AsmCalls(); + fresh_portable = EachOperationOnAFreshThread(ek, dk, ct); } - std::printf("active=%.*s detected_calls=%lu portable_calls=%lu equal=%d ok=%d\n", int(active.size()), active.data(), - detected_calls, portable_calls, int(detected == portable), int(!detected.empty())); + std::printf("active=%.*s detected_calls=%lu portable_calls=%lu equal=%d ok=%d " + "fresh_detected=%lu,%lu,%lu,%lu fresh_portable=%lu,%lu,%lu,%lu\n", + int(active.size()), active.data(), detected_calls, portable_calls, int(detected == portable), + int(!detected.empty() && inputs), fresh_detected[0], fresh_detected[1], fresh_detected[2], fresh_detected[3], + fresh_portable[0], fresh_portable[1], fresh_portable[2], fresh_portable[3]); } """ +# The entry points EachOperationOnAFreshThread runs, in its order. +OPERATIONS = ("KeyGen", "CheckPublicKey", "Encaps", "Decaps") + +# Test-only edits that let qbit's AArch64 glue build for AArch64 ELF, which +# release builds refuse (src/crypto/mlkem_config.h: no BTI landing pads), so +# that it can run under qemu-aarch64 on a Linux host. Each must match exactly once. +AARCH64_ELF_CONFIG_EDITS = ( + ("#if defined(MLK_SYS_AARCH64) && defined(__ELF__)\n" + "#error \"mlkem-native AArch64 assembly has no BTI landing pads or GNU property note; AArch64 ELF builds are portable only.\"\n" + "#endif\n", ""), + ("(defined(MLK_SYS_AARCH64) && defined(MLK_SYS_AARCH64_NEON) && defined(MLK_SYS_APPLE))", + "(defined(MLK_SYS_AARCH64) && defined(MLK_SYS_AARCH64_NEON))"), +) +# The negative control: upstream's single-lane Keccak glue, which runs its +# assembly without asking the capability hook, in place of qbit's mirror. +UPSTREAM_X1_EDIT = ( + re.compile(r"#define MLK_USE_NATIVE_FIPS202_X1\n.*?#endif // !__ASSEMBLER__\n", re.S), + "#include \n", +) + + +def edited(source: Path, edits: tuple) -> str: + text = source.read_text(encoding="utf8") + for old, new in edits: + count = len(old.findall(text)) if isinstance(old, re.Pattern) else text.count(old) + if count != 1: + raise AssertionError( + f"FAIL: test edit matched {count} times in {source}\n" + f"Cause: {source.name} changed, so the test can no longer derive its AArch64 ELF build\n" + f"Fix: update the edits in {Path(__file__).name} to the new text: {old if isinstance(old, str) else old.pattern!r}") + text = old.sub(new, text) if isinstance(old, re.Pattern) else text.replace(old, new) + return text + class MlkemOverrideCoverageTest(unittest.TestCase): - """The portable override reaches every assembly routine of the x86_64 backend. + """The portable override reaches every assembly routine of each native backend. Every global symbol the assembly object defines is wrapped (ld --wrap) by a counter. A missed entry point would still give identical outputs, so only - counting the calls can show it. + counting the calls can show it. Each wrapper entry point is also run alone + on a fresh thread, which shows one that runs the library without taking + the override first. + + x86_64 runs on x86_64 hosts. AArch64 is cross-built as a static Linux + binary and run under qemu-aarch64; release builds enable it only on macOS, + so the test builds a copy of src/crypto/mlkem_config.h without its AArch64 + ELF refusal. CI requires the AArch64 tools through + MLKEM_POLICY_REQUIRED_COMPILERS (aarch64-linux-gnu-g++ and qemu-aarch64). """ - def test_override_reaches_every_assembly_routine(self) -> None: + def build_harness(self, out: Path, *, cc: str, cxx: str, nm: str, args: int, overrides: dict[str, str], + link_flags: tuple[str, ...] = ()) -> Path: + """Compile the library, qbit's glue and the counting harness; return the binary.""" + src = REPO_ROOT / "src" + include = out / "include" + for name, text in overrides.items(): + (include / "crypto").mkdir(parents=True, exist_ok=True) + (include / "crypto" / name).write_text(text, encoding="utf8") + # Overridden glue headers come first; qbit's own files fill in the rest. + flags = ['-DMLK_CONFIG_FILE="crypto/mlkem_config.h"', "-DQBIT_MLKEM_NATIVE", f"-I{include}", f"-I{src}", + f"-I{src / 'mlkem-native'}", "-O1"] + + def compile_(compiler: str, source: Path, *extra: str) -> Path: + obj = out / (source.name + ".o") + result = run([compiler, *flags, *extra, "-c", str(source), "-o", str(obj)]) + self.assertEqual(result.returncode, 0, result.stderr) + return obj + + objects = [ + compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native.c"), + compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native_asm.S"), + compile_(cc, src / "crypto" / "mlkem_backend.c"), + ] + asm_symbols = sorted(line.split()[2] for line in run([nm, "--defined-only", "-g", str(objects[1])]).stdout.splitlines() + if len(line.split()) == 3 and line.split()[1] == "T") + self.assertGreater(len(asm_symbols), 0, "no assembly routines: is the native backend compiled?") + # Every routine takes only integer or pointer arguments, at most as many + # as the ABI passes in registers (args), so a generic forwarder keeps them. + params = ", ".join("void*" for _ in range(args)) + named = ", ".join(f"void* a{i}" for i in range(args)) + forwarded = ", ".join(f"a{i}" for i in range(args)) + wrappers = ["#include ", "#include ", "static std::atomic g_calls{0};", 'extern "C" {'] + for symbol in asm_symbols: + wrappers.append(f"uint64_t __real_{symbol}({params});") + wrappers.append(f"uint64_t __wrap_{symbol}({named}) {{ ++g_calls; return __real_{symbol}({forwarded}); }}") + wrappers += ["}", "unsigned long AsmCalls() { return g_calls.exchange(0); }"] + (out / "wrap.cpp").write_text("\n".join(wrappers) + "\n", encoding="utf8") + (out / "harness.cpp").write_text(COVERAGE_HARNESS, encoding="utf8") + for source in (src / "crypto" / "mlkem.cpp", src / "compat" / "cpu_features.cpp", src / "support" / "cleanse.cpp", + out / "harness.cpp", out / "wrap.cpp"): + objects.append(compile_(cxx, source, "-std=c++20", "-pthread")) + binary = out / "harness" + link = run([cxx, "-pthread", *link_flags, "-o", str(binary), *map(str, objects), + *(f"-Wl,--wrap={symbol}" for symbol in asm_symbols)]) + self.assertEqual(link.returncode, 0, link.stderr) + return binary + + def run_harness(self, command: list[str]) -> dict[str, str]: + result = run(command) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + values = dict(item.split("=", 1) for item in result.stdout.split()) + self.assertEqual(values["ok"], "1", result.stdout) + self.assertEqual(values["equal"], "1", "the backends disagree") + return values + + def check_coverage(self, values: dict[str, str], native: str, mirror_hint: str) -> None: + self.assertEqual(values["active"], native, values) + self.assertGreater(int(values["detected_calls"]), 0, values) + self.assertEqual(values["portable_calls"], "0", + f"FAIL: assembly ran {values['portable_calls']} times with portable C forced\n" + "Cause: a native entry point does not ask mlk_sys_check_capability\n" + f"Fix: mirror its dispatch glue with a capability check, {mirror_hint}") + fresh_detected = [int(count) for count in values["fresh_detected"].split(",")] + fresh_portable = [int(count) for count in values["fresh_portable"].split(",")] + for operation, native_calls, portable_calls in zip(OPERATIONS, fresh_detected, fresh_portable): + # Without native calls of its own, an operation's portable count proves nothing. + self.assertGreater(native_calls, 0, f"{operation} runs no assembly natively: {values}") + self.assertEqual(portable_calls, 0, + f"FAIL: mlkem::{operation} ran assembly {portable_calls} times on a fresh thread with portable C forced\n" + "Cause: it enters the library without BeginOperation(), so this thread's copy of the override is stale\n" + "Fix: call BeginOperation() first in every entry point of src/crypto/mlkem.cpp") + + def test_x86_64_override_reaches_every_assembly_routine(self) -> None: if host_arch() != "x86_64": self.skipTest("the x86_64 backend runs only on x86_64 hosts") cc, cxx, nm = shutil.which("cc"), shutil.which("c++"), shutil.which("nm") if not (cc and cxx and nm): self.skipTest("cc, c++ and nm are required") assert cc is not None and cxx is not None and nm is not None + with tempfile.TemporaryDirectory() as tmp: + binary = self.build_harness(Path(tmp), cc=cc, cxx=cxx, nm=nm, args=6, overrides={}) + values = self.run_harness([str(binary)]) + if values["active"] != "x86_64-avx2": + # A CPU without the extensions runs portable C only: nothing to count. + self.assertEqual(values["detected_calls"], "0", values) + self.assertEqual(values["portable_calls"], "0", values) + self.skipTest("this CPU cannot run the x86_64 backend") + self.check_coverage(values, "x86_64-avx2", "as for AArch64's x1 Keccak in src/crypto/mlkem_fips202_backend.h") + + def aarch64_tools(self) -> tuple[str, str, str, str]: + tools = [] + for executable in ("aarch64-linux-gnu-gcc", "aarch64-linux-gnu-g++", "aarch64-linux-gnu-nm", "qemu-aarch64"): + path = shutil.which(executable) + if path is None: + skip_missing_compiler(self, executable) + assert path is not None + tools.append(path) + return tools[0], tools[1], tools[2], tools[3] + + def test_aarch64_override_reaches_every_assembly_routine(self) -> None: + cc, cxx, nm, qemu = self.aarch64_tools() src = REPO_ROOT / "src" - flags = ['-DMLK_CONFIG_FILE="crypto/mlkem_config.h"', "-DQBIT_MLKEM_NATIVE", f"-I{src}", f"-I{src / 'mlkem-native'}", "-O1"] + config = edited(src / "crypto" / "mlkem_config.h", AARCH64_ELF_CONFIG_EDITS) with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) - - def compile_(compiler: str, source: Path, *extra: str) -> Path: - obj = out / (source.name + ".o") - result = run([compiler, *flags, *extra, "-c", str(source), "-o", str(obj)]) - self.assertEqual(result.returncode, 0, result.stderr) - return obj - - objects = [ - compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native.c"), - compile_(cc, src / "mlkem-native" / "mlkem" / "mlkem_native_asm.S"), - compile_(cc, src / "crypto" / "mlkem_backend.c"), - ] - asm_symbols = sorted(line.split()[2] for line in run([nm, "--defined-only", "-g", str(objects[1])]).stdout.splitlines() - if len(line.split()) == 3 and line.split()[1] == "T") - self.assertGreater(len(asm_symbols), 0, "no assembly routines: is the native backend compiled?") - wrappers = ["#include ", "#include ", "static std::atomic g_calls{0};", 'extern "C" {'] - for symbol in asm_symbols: - # Every routine takes at most six integer or pointer arguments, which the - # SysV ABI passes in registers, so a generic forwarder preserves them. - wrappers.append(f"uint64_t __real_{symbol}(void*, void*, void*, void*, void*, void*);") - wrappers.append(f"uint64_t __wrap_{symbol}(void* a, void* b, void* c, void* d, void* e, void* f) " - f"{{ ++g_calls; return __real_{symbol}(a, b, c, d, e, f); }}") - wrappers += ["}", "unsigned long AsmCalls() { return g_calls.exchange(0); }"] - (out / "wrap.cpp").write_text("\n".join(wrappers) + "\n", encoding="utf8") - (out / "harness.cpp").write_text(COVERAGE_HARNESS, encoding="utf8") - for source in (src / "crypto" / "mlkem.cpp", src / "compat" / "cpu_features.cpp", src / "support" / "cleanse.cpp", - out / "harness.cpp", out / "wrap.cpp"): - objects.append(compile_(cxx, source, "-std=c++20")) - binary = out / "harness" - link = run([cxx, "-o", str(binary), *map(str, objects), *(f"-Wl,--wrap={symbol}" for symbol in asm_symbols)]) - self.assertEqual(link.returncode, 0, link.stderr) - result = run([str(binary)]) - self.assertEqual(result.returncode, 0, result.stderr) - values = dict(item.split("=", 1) for item in result.stdout.split()) - self.assertEqual(values["ok"], "1", result.stdout) - self.assertEqual(values["equal"], "1", "the backends disagree") - self.assertEqual(values["portable_calls"], "0", - f"FAIL: assembly ran {values['portable_calls']} times with portable C forced\n" - "Cause: a native entry point does not ask mlk_sys_check_capability\n" - "Fix: mirror its dispatch glue with a capability check, as for AArch64's x1 Keccak in src/crypto/mlkem_fips202_backend.h") - if values["active"] == "x86_64-avx2": - self.assertGreater(int(values["detected_calls"]), 0, result.stdout) - else: - self.assertEqual(values["detected_calls"], "0", result.stdout) + out = Path(tmp) / "qbit" + out.mkdir() + binary = self.build_harness(out, cc=cc, cxx=cxx, nm=nm, args=8, overrides={"mlkem_config.h": config}, + link_flags=("-static",)) + values = self.run_harness([qemu, str(binary)]) + self.check_coverage(values, "aarch64-neon", + "as src/crypto/mlkem_fips202_backend.h does for upstream's x1_scalar.h") + + # Negative control: with upstream's x1 glue in place of qbit's + # mirror, forced portable runs leak assembly calls, and the count sees them. + upstream = Path(tmp) / "upstream-x1" + upstream.mkdir() + fips202 = edited(src / "crypto" / "mlkem_fips202_backend.h", (UPSTREAM_X1_EDIT,)) + binary = self.build_harness(upstream, cc=cc, cxx=cxx, nm=nm, args=8, + overrides={"mlkem_config.h": config, "mlkem_fips202_backend.h": fips202}, + link_flags=("-static",)) + control = self.run_harness([qemu, str(binary)]) + self.assertGreater(int(control["portable_calls"]), 0, + f"FAIL: upstream's x1 glue leaked no assembly calls with portable C forced: {control}\n" + "Cause: the counter no longer sees the x1 Keccak routine, so this test cannot catch a missing capability check\n" + "Fix: check that the --wrap list covers every global symbol of mlkem_native_asm.S") if __name__ == "__main__": diff --git a/doc/subtrees/mlkem-native.md b/doc/subtrees/mlkem-native.md index 0079a05670..a2441f8dcd 100644 --- a/doc/subtrees/mlkem-native.md +++ b/doc/subtrees/mlkem-native.md @@ -64,9 +64,14 @@ says what was chosen and why: | `SANITIZERS` contains `memory` | portable (MSan cannot see assembly writes) | configure error | | anything else (riscv64, powerpc64, armv7, s390x, i686, ...) | portable | configure error | -`ci/checks/test_mlkem_build_policy.py` also links the glue with every x86_64 -assembly routine wrapped by a call counter, and fails if any of them runs -while portable C is forced. +`ci/checks/test_mlkem_build_policy.py` also links the glue with every +assembly routine wrapped by a call counter, and fails if any of them runs while +portable C is forced, or when an entry point run alone on a new thread uses +assembly although portable C is forced (it entered the library without taking +the override). It checks the x86_64 backend natively and the AArch64 backend +as a static binary under `qemu-aarch64`, built from a copy of +`src/crypto/mlkem_config.h` without its AArch64 ELF refusal. CI's build smoke +job requires both. ### x86_64 instruction set @@ -165,6 +170,11 @@ dispatch glue (the same macros and the same proved assembly routine) with a capability check added. Every other native entry point in the x86_64 and AArch64 backends checks the hook itself. +The build-policy test's AArch64 case shows the mirror at work: the harness +makes 252 assembly calls natively and none with portable C forced. Its negative +control builds upstream's `x1_scalar.h` in place of the mirror, and then 221 +assembly calls leak into a forced-portable run. + ## Update Procedure Run in a qbit worktree: From d77ee3518650cc556e0deb24d6faa80e772a28d5 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 12:45:17 -0400 Subject: [PATCH 13/20] test: cover each ML-KEM entry point's override and its per-thread copy Two wrapper mutations passed every test. An entry point without BeginOperation() went unnoticed because ForEachBackend calls GetBackendNames() on the same thread first, which takes the override for it. A shared (not thread_local) copy of the override went unnoticed because only one thread ran operations. every_entry_point_takes_the_override runs each entry point alone on a new thread with portable C forced, then asks the capability hooks what that thread's copy says. override_copy_is_per_thread lets one thread take the override set while another holds it unset, and checks that the first thread's copy did not change. override_changed_by_another_thread now runs operations on two threads at once, so a shared copy also gives wrong outputs on x86_64 and a data race under ThreadSanitizer. Co-Authored-By: Claude Opus 5.5 --- src/test/mlkem_tests.cpp | 138 +++++++++++++++++++++++++++++++++------ 1 file changed, 117 insertions(+), 21 deletions(-) diff --git a/src/test/mlkem_tests.cpp b/src/test/mlkem_tests.cpp index 6e6a6edc7a..546cdb5473 100644 --- a/src/test/mlkem_tests.cpp +++ b/src/test/mlkem_tests.cpp @@ -29,12 +29,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include namespace mlkem { @@ -87,6 +89,12 @@ bool AllBytesAre(std::span bytes, uint8_t value) return std::ranges::all_of(bytes, [&](uint8_t b) { return b == value; }); } +/** Whether the capability hooks would let native code run now, on this thread, per its copy of the override. */ +bool HooksAllowNative() +{ + return qbit_mlkem_has_avx2() != 0 || qbit_mlkem_has_neon() != 0; +} + /** Run `check` with the active backend, then again with portable C forced. */ void ForEachBackend(const std::function& check) { @@ -563,6 +571,88 @@ BOOST_AUTO_TEST_CASE(backend_selection) BOOST_CHECK_EQUAL(GetBackendNames().keccak, active.keccak); } +BOOST_AUTO_TEST_CASE(every_entry_point_takes_the_override) +{ + // Each entry point fixes the backend for its operation at entry, from its + // own thread's copy of the override. Run each alone on a new thread, whose + // copy starts unset, and then ask the hooks what that copy says. The + // hooks can say native only where this CPU and build allow it. + (void)GetBackendNames(); + const bool native_possible{HooksAllowNative()}; + BOOST_TEST_MESSAGE("hooks can allow native code here: " << native_possible); + + std::array seed; + seed.fill(0x11); + const std::array coins{}; + PublicKey ek; + DecapsulationKey dk; + Ciphertext ct; + SharedSecret ss; + BOOST_REQUIRE_EQUAL(KeyGen(seed, ek, dk), Error::NONE); + BOOST_REQUIRE_EQUAL(Encaps(ek, coins, ct, ss), Error::NONE); + const std::vector>> entry_points{ + {"KeyGen", [&] { PublicKey e; DecapsulationKey d; (void)KeyGen(seed, e, d); }}, + {"CheckPublicKey", [&] { (void)CheckPublicKey(ek); }}, + {"Encaps", [&] { Ciphertext c; SharedSecret s; (void)Encaps(ek, coins, c, s); }}, + {"Decaps", [&] { SharedSecret s; (void)Decaps(dk, ct, s); }}, + {"GetBackendNames", [] { (void)GetBackendNames(); }}, + }; + for (const auto& [name, entry_point] : entry_points) { + bool native_while_forced{true}; + bool native_after{!native_possible}; + std::thread{[&] { + { + ForcePortableForTesting portable; + entry_point(); + native_while_forced = HooksAllowNative(); + } + entry_point(); + native_after = HooksAllowNative(); + }}.join(); + BOOST_CHECK_MESSAGE(!native_while_forced, "FAIL: after mlkem::" << name << " on a new thread with portable C forced, the hooks allow native code. " + << "Cause: it entered the library without taking the override, so native code can run. " + << "Fix: call BeginOperation() first in every entry point of src/crypto/mlkem.cpp."); + BOOST_CHECK_MESSAGE(native_after == native_possible, "FAIL: after mlkem::" << name << " with the override cleared, the hooks allow native code: " + << native_after << ", expected " << native_possible << "."); + } +} + +BOOST_AUTO_TEST_CASE(override_copy_is_per_thread) +{ + // Thread A takes the override unset; thread B then takes it set. A's copy + // must not change: a copy shared between threads would switch A's running + // operation to the other backend. + (void)GetBackendNames(); + const bool native_possible{HooksAllowNative()}; + std::promise a_took, b_took; + std::future a_took_future{a_took.get_future()}; + std::future b_took_future{b_took.get_future()}; + bool a_before{!native_possible}; + bool a_after{!native_possible}; + std::thread a{[&] { + (void)GetBackendNames(); + a_before = HooksAllowNative(); + a_took.set_value(); + b_took_future.wait(); + a_after = HooksAllowNative(); + }}; + a_took_future.wait(); + bool b_native{true}; + { + ForcePortableForTesting portable; + std::thread{[&] { + (void)GetBackendNames(); + b_native = HooksAllowNative(); + }}.join(); + } + b_took.set_value(); + a.join(); + BOOST_CHECK(!b_native); + BOOST_CHECK_EQUAL(a_before, native_possible); + BOOST_CHECK_MESSAGE(a_after == a_before, "FAIL: thread A's override changed when thread B took it. " + << "Cause: the copy the hooks read is shared between threads. Fix: keep t_force_portable thread_local in src/crypto/mlkem.cpp."); +} + BOOST_AUTO_TEST_CASE(override_changed_by_another_thread) { const BackendNames before{GetBackendNames()}; @@ -578,9 +668,10 @@ BOOST_AUTO_TEST_CASE(override_changed_by_another_thread) BOOST_REQUIRE_EQUAL(KeyGen(seed, expected_ek, expected_dk), Error::NONE); BOOST_REQUIRE_EQUAL(Encaps(expected_ek, coins, expected_ct, expected_ss), Error::NONE); - // Another thread flips the override as fast as it can while this one runs - // operations. Each operation must finish on the backend it started with: - // on x86_64, one that mixed backends would give wrong outputs. + // A third thread flips the override as fast as it can while two threads + // run operations at once. Each operation must finish on the backend it + // started with: on x86_64, one that mixed backends would give wrong + // outputs, and under ThreadSanitizer a shared copy of the override races. std::atomic_bool stop{false}; std::atomic flips{0}; std::thread flipper{[&] { @@ -590,28 +681,33 @@ BOOST_AUTO_TEST_CASE(override_changed_by_another_thread) } }}; while (flips.load(std::memory_order_relaxed) == 0) std::this_thread::yield(); - int wrong{0}; - int portable_seen{0}; + std::atomic wrong{0}; + std::atomic portable_seen{0}; constexpr int ROUNDS{50}; - for (int round{0}; round < ROUNDS; ++round) { - PublicKey ek; - DecapsulationKey dk; - Ciphertext ct; - SharedSecret ss; - SharedSecret decapsulated; - const bool ok{KeyGen(seed, ek, dk) == Error::NONE && Encaps(ek, coins, ct, ss) == Error::NONE && - Decaps(dk, ct, decapsulated) == Error::NONE}; - if (!ok || ek != expected_ek || !std::ranges::equal(dk.Bytes(), expected_dk.Bytes()) || ct != expected_ct || - !std::ranges::equal(ss.Bytes(), expected_ss.Bytes()) || !std::ranges::equal(decapsulated.Bytes(), expected_ss.Bytes())) { - ++wrong; + const auto run_rounds{[&] { + for (int round{0}; round < ROUNDS; ++round) { + PublicKey ek; + DecapsulationKey dk; + Ciphertext ct; + SharedSecret ss; + SharedSecret decapsulated; + const bool ok{KeyGen(seed, ek, dk) == Error::NONE && Encaps(ek, coins, ct, ss) == Error::NONE && + Decaps(dk, ct, decapsulated) == Error::NONE}; + if (!ok || ek != expected_ek || !std::ranges::equal(dk.Bytes(), expected_dk.Bytes()) || ct != expected_ct || + !std::ranges::equal(ss.Bytes(), expected_ss.Bytes()) || !std::ranges::equal(decapsulated.Bytes(), expected_ss.Bytes())) { + ++wrong; + } + if (GetBackendNames().arith == "portable") ++portable_seen; } - if (GetBackendNames().arith == "portable") ++portable_seen; - } + }}; + std::thread second{run_rounds}; + run_rounds(); + second.join(); stop.store(true, std::memory_order_relaxed); flipper.join(); - BOOST_TEST_MESSAGE(flips.load() << " flips; portable active at " << portable_seen << " of " << ROUNDS << " checks"); - BOOST_CHECK_MESSAGE(wrong == 0, "FAIL: " << wrong << " of " << ROUNDS << " rounds gave wrong outputs while another thread changed the override. " - << "Cause: an operation mixed backends. Fix: the capability hooks in src/crypto/mlkem.cpp must read the copy taken at entry."); + BOOST_TEST_MESSAGE(flips.load() << " flips; portable active at " << portable_seen.load() << " of " << 2 * ROUNDS << " checks"); + BOOST_CHECK_MESSAGE(wrong.load() == 0, "FAIL: " << wrong.load() << " of " << 2 * ROUNDS << " rounds gave wrong outputs while another thread changed the override. " + << "Cause: an operation mixed backends. Fix: the capability hooks in src/crypto/mlkem.cpp must read the thread_local copy taken at entry."); BOOST_CHECK_EQUAL(GetBackendNames().arith, before.arith); } From ad56a2612d664974a64e97f81b623aa906ca55e1 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 12:47:06 -0400 Subject: [PATCH 14/20] fuzz: check that the ML-KEM portable replay runs portable C CI replays the mlkem and mlkem_backend_diff targets a second time with MLKEM_FORCE_PORTABLE=1, but nothing checked that the binary honored it: any other value, such as "true", silently selected native code, and the second replay would have repeated the first. Initialize now refuses any value but unset, empty, 0 or 1, prints the active backend once, and aborts when MLKEM_FORCE_PORTABLE=1 but the active backend is not portable/portable. A host without native code also shows in the log, where mlkem_backend_diff compares portable C with itself. Co-Authored-By: Claude Opus 5.5 --- doc/development/fuzzing.md | 8 +++++--- src/test/fuzz/mlkem.cpp | 22 +++++++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/doc/development/fuzzing.md b/doc/development/fuzzing.md index 477bbc70ba..4d43732d84 100644 --- a/doc/development/fuzzing.md +++ b/doc/development/fuzzing.md @@ -57,9 +57,11 @@ selected (including through `--exclude`), has no regular input files, or is not reported as replayed by the fuzz executable. The runner prints the number of input files present and replayed for each target. -The ML-KEM targets use native code where the CPU has it. CI replays `mlkem` and -`mlkem_backend_diff` a second time with `MLKEM_FORCE_PORTABLE=1`, which makes -them run portable C throughout. +The ML-KEM targets use native code where the CPU has it, and print the backend +they run at start-up. CI replays `mlkem` and `mlkem_backend_diff` a second time +with `MLKEM_FORCE_PORTABLE=1`, which makes them run portable C throughout; they +abort if portable C is not what runs, and on any value other than unset, `0` or +`1`. With a libFuzzer build, `--mutate_min_time=` replaces the single replay with a mutation phase of that many seconds per target. The phase starts diff --git a/src/test/fuzz/mlkem.cpp b/src/test/fuzz/mlkem.cpp index 417b0d1ab6..b622ed25c2 100644 --- a/src/test/fuzz/mlkem.cpp +++ b/src/test/fuzz/mlkem.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -22,9 +23,24 @@ namespace { void Initialize() { - // MLKEM_FORCE_PORTABLE=1 runs every operation on portable C. - const char* force_portable{std::getenv("MLKEM_FORCE_PORTABLE")}; - InitializeRuntime(force_portable != nullptr && std::string_view{force_portable} == "1"); + // MLKEM_FORCE_PORTABLE=1 runs every operation on portable C; unset, empty + // or 0 runs the detected backend. Any other value is refused, so that a + // misspelled setting cannot turn CI's portable replay into a second + // native one, and the portable replay checks that portable C is what runs. + const char* env{std::getenv("MLKEM_FORCE_PORTABLE")}; + const std::string_view setting{env != nullptr ? env : ""}; + if (setting != "" && setting != "0" && setting != "1") { + std::cerr << "MLKEM_FORCE_PORTABLE must be unset, 0 or 1, not '" << setting << "'" << std::endl; + std::abort(); + } + InitializeRuntime(/*force_portable=*/setting == "1"); + const BackendNames active{GetBackendNames()}; + std::cerr << "ML-KEM backend: " << active.arith << "/" << active.keccak + << (setting == "1" ? " (MLKEM_FORCE_PORTABLE=1)" : "") << std::endl; + if (setting == "1" && (active.arith != "portable" || active.keccak != "portable")) { + std::cerr << "MLKEM_FORCE_PORTABLE=1, but the active backend is not portable" << std::endl; + std::abort(); + } } template From c1eb233417751f2ec21e746782c2db8ee66eeac3 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 12:53:39 -0400 Subject: [PATCH 15/20] test: cover SHA256's SSSE3 requirement with synthetic CPUs SHA256AutoDetect requires SSSE3 besides SSE4.1 before it selects its SSE4, SSE4.1 or SHA-NI code, all of which execute SSSE3 instructions. Nothing tested that: dropping the SSSE3 term passed every test, because real SSE4.1 hardware always has SSSE3. An overload of SHA256AutoDetect takes the CPUID and XGETBV functions, as cpu_features::DetectX86Features does, and the existing one passes the real ones, so the queries and the selection are unchanged. A cpu_features_tests case runs it on synthetic CPUs derived from this one: unmasked it selects what this CPU gets, and with SSSE3 or SSE4.1 masked the standard code. Co-Authored-By: Claude Opus 5.5 --- src/crypto/sha256.cpp | 21 ++++++++++++++++----- src/crypto/sha256.h | 12 ++++++++++++ src/test/cpu_features_tests.cpp | 31 +++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 3fcaa22ea7..40250e42ce 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -571,9 +571,9 @@ bool SelfTest() { #if !defined(DISABLE_OPTIMIZED_SHA256) #if (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) /** Check whether the OS has enabled AVX registers. */ -bool AVXEnabled() +bool AVXEnabled(sha256_implementation::XGetBVFn xgetbv) { - return (XGetBV(0) & 6) == 6; + return (xgetbv(0) & 6) == 6; } #endif #endif // DISABLE_OPTIMIZED_SHA256 @@ -581,6 +581,17 @@ bool AVXEnabled() std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation) +{ +#if defined(HAVE_GETCPUID) + return SHA256AutoDetect(use_implementation, GetCPUID, XGetBV); +#else + return SHA256AutoDetect(use_implementation, nullptr, nullptr); +#endif +} + +std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation, + [[maybe_unused]] sha256_implementation::CpuidFn cpuid, + [[maybe_unused]] sha256_implementation::XGetBVFn xgetbv) { std::string ret = "standard"; Transform = sha256::Transform; @@ -599,7 +610,7 @@ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implem [[maybe_unused]] bool enabled_avx = false; uint32_t eax, ebx, ecx, edx; - GetCPUID(1, 0, eax, ebx, ecx, edx); + cpuid(1, 0, eax, ebx, ecx, edx); if (use_implementation & sha256_implementation::USE_SSE4) { // Fixes an upstream Bitcoin Core bug, which checks SSE4.1 only. The // SSE4 code and the SSE4.1 and SHA-NI code chosen under this check @@ -611,10 +622,10 @@ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implem have_xsave = (ecx >> 27) & 1; have_avx = (ecx >> 28) & 1; if (have_xsave && have_avx) { - enabled_avx = AVXEnabled(); + enabled_avx = AVXEnabled(xgetbv); } if (have_sse4) { - GetCPUID(7, 0, eax, ebx, ecx, edx); + cpuid(7, 0, eax, ebx, ecx, edx); if (use_implementation & sha256_implementation::USE_AVX2) { have_avx2 = (ebx >> 5) & 1; } diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index f8a3b2c930..dc71b603cd 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -36,6 +36,10 @@ enum UseImplementation : uint8_t { USE_SSE4_AND_SHANI = USE_SSE4 | USE_SHANI, USE_ALL = USE_SSE4 | USE_AVX2 | USE_SHANI, }; + +/** The x86 CPUID and XGETBV queries, as compat/cpuid.h's GetCPUID and XGetBV. */ +using CpuidFn = void (*)(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d); +using XGetBVFn = uint64_t (*)(uint32_t index); } /** Autodetect the best available SHA256 implementation. @@ -43,6 +47,14 @@ enum UseImplementation : uint8_t { */ std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation = sha256_implementation::USE_ALL); +/** SHA256AutoDetect with the x86 CPUID and XGETBV queries answered by the + * given functions, so that tests can present synthetic CPUs; other targets + * ignore them. The selected code runs its self-test, so a synthetic CPU must + * not claim an extension this one lacks. + */ +std::string SHA256AutoDetect(sha256_implementation::UseImplementation use_implementation, + sha256_implementation::CpuidFn cpuid, sha256_implementation::XGetBVFn xgetbv); + /** Compute multiple double-SHA256's of 64-byte blobs. * output: pointer to a blocks*32 byte output buffer * input: pointer to a blocks*64 byte input buffer diff --git a/src/test/cpu_features_tests.cpp b/src/test/cpu_features_tests.cpp index ca9e3effbe..9d35408f4a 100644 --- a/src/test/cpu_features_tests.cpp +++ b/src/test/cpu_features_tests.cpp @@ -6,10 +6,12 @@ #include #include +#include #include #include +#include #include using namespace cpu_features; @@ -183,6 +185,35 @@ BOOST_AUTO_TEST_CASE(mlkem_x86_native_matrix) BOOST_CHECK(!HasMlkemX86Native(DetectFake(without_leaf1(LEAF1_ECX_POPCNT)))); } +BOOST_AUTO_TEST_CASE(sha256_x86_code_needs_ssse3) +{ +#if defined(HAVE_GETCPUID) + // SHA256AutoDetect, given synthetic CPUs derived from this one so that the + // code it selects can run its self-test here. Its SSE4, SSE4.1 4-way and + // SHA-NI code all execute SSSE3 instructions (pshufb), so a CPU that + // masks SSSE3 alone, as a virtual machine can, must get the standard code, + // like one without SSE4.1. + const X86Features host{DetectX86Features()}; + const auto detect_masked{[&](uint32_t leaf1_ecx_mask) { + g_cpu = FakeCpu{}; + g_cpu.features = host; + g_cpu.features.leaf1_ecx &= ~leaf1_ecx_mask; + return SHA256AutoDetect(sha256_implementation::USE_ALL, FakeCpuid, FakeXGetBV); + }}; + const std::string host_choice{SHA256AutoDetect()}; + BOOST_TEST_MESSAGE("SHA256 on this CPU: " << host_choice); + // Unmasked, the synthetic CPU gets what this one does. + BOOST_CHECK_EQUAL(detect_masked(0), host_choice); + BOOST_CHECK_EQUAL(detect_masked(LEAF1_ECX_SSSE3), "standard"); + BOOST_CHECK_EQUAL(detect_masked(LEAF1_ECX_SSE41), "standard"); + // Restore this CPU's selection. + BOOST_CHECK_EQUAL(SHA256AutoDetect(), host_choice); +#else + // Other targets ignore the x86 queries. + BOOST_CHECK_EQUAL(SHA256AutoDetect(sha256_implementation::USE_ALL, nullptr, nullptr), SHA256AutoDetect()); +#endif +} + BOOST_AUTO_TEST_CASE(usable_avx2_host) { const X86Features detected{DetectX86Features()}; From ba3e545e4a31dc02b82476087c975baa86213aee Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 12:54:35 -0400 Subject: [PATCH 16/20] doc: say exactly where ML-KEM native code runs The wrapper's header said native code runs on "x86_64 with AVX2, AArch64", and the WITH_MLKEM_NATIVE help text spoke of "the x86_64 and AArch64 ML-KEM assembly". AArch64 runs native code only on macOS (AArch64 ELF builds stay portable to keep their BTI and PAC marking), and x86_64 also needs SSSE3, SSE4.1, POPCNT and BMI2, as cpu_features::HasMlkemX86Native checks. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 +- src/crypto/mlkem.h | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 720f071bbb..30f7cb1e77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,7 +136,7 @@ if(WITH_USDT) find_package(USDT MODULE REQUIRED) endif() -set(WITH_MLKEM_NATIVE "AUTO" CACHE STRING "Build the x86_64 and AArch64 ML-KEM assembly, selected at run time: AUTO, ON or OFF.") +set(WITH_MLKEM_NATIVE "AUTO" CACHE STRING "Build ML-KEM native code for x86_64 (GCC or Clang) and macOS arm64, used when the CPU allows it: AUTO, ON or OFF.") set_property(CACHE WITH_MLKEM_NATIVE PROPERTY STRINGS AUTO ON OFF) option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." ON) diff --git a/src/crypto/mlkem.h b/src/crypto/mlkem.h index 92b7e2fff9..ab2cc03aaa 100644 --- a/src/crypto/mlkem.h +++ b/src/crypto/mlkem.h @@ -16,9 +16,11 @@ /** * ML-KEM-1024 (FIPS 203), backed by the vendored mlkem-native library. * - * Native code runs where it is supported and the CPU allows it (x86_64 with - * AVX2, AArch64), and portable C everywhere else. The library is built into - * bitcoin_node only. + * Native code runs where it is built and the CPU allows it: x86_64 with AVX2, + * SSSE3, SSE4.1, POPCNT and BMI2 (and the OS saving AVX state), and AArch64 on + * macOS. Everything else runs portable C, AArch64 ELF (Linux, BSD) included, + * whose builds leave the assembly out to keep their BTI and PAC marking. See + * doc/subtrees/mlkem-native.md. The library is built into bitcoin_node only. * * Fixed-size spans make wrong-sized calls impossible; callers check network * lengths before constructing them. Entropy is the caller's job: key From 4ef6339bdbdd36ec87c8e1b16a44367dc9a3d7c3 Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 12:55:54 -0400 Subject: [PATCH 17/20] contrib: keep the clone unshallow when vendoring mlkem-native update-mlkem-native.sh fetched the upstream release with --depth=1 into the qbit repository itself. That writes .git/shallow, so a full clone then reported itself as shallow to git and every tool that asks. Fetch into a temporary bare repository instead, and read the commit, the object ids and the archive from there. Staging and the byte-for-byte check still run in the qbit repository. Re-vendoring v2.0.0 by tag and by commit leaves no diff, and the clone stays complete. Co-Authored-By: Claude Opus 5.5 --- contrib/devtools/update-mlkem-native.sh | 19 ++++++++++++++----- doc/subtrees/mlkem-native.md | 3 ++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/contrib/devtools/update-mlkem-native.sh b/contrib/devtools/update-mlkem-native.sh index 10bf8de331..34212b7c6e 100755 --- a/contrib/devtools/update-mlkem-native.sh +++ b/contrib/devtools/update-mlkem-native.sh @@ -71,13 +71,22 @@ if [[ -z "${REMOTE_URL}" || -z "${REMOTE_REF}" ]]; then "restore ${PIN_FILE} from git, or set REMOTE_URL and pass a tag" fi +# Fetch into a temporary repository: a depth-1 fetch into this one would +# write .git/shallow and turn it into a shallow clone. +upstream_git="$(mktemp -d "${TMPDIR:-/tmp}/update-mlkem-native.XXXXXX")" +trap 'rm -rf -- "${upstream_git}"' EXIT +git init --quiet --bare "${upstream_git}" +upstream() { + git --git-dir="${upstream_git}" "$@" +} + echo "Fetching mlkem-native ${REMOTE_REF} from ${REMOTE_URL}" -if ! git fetch --quiet --depth=1 --no-tags "${REMOTE_URL}" "${REMOTE_REF}"; then +if ! upstream fetch --quiet --depth=1 --no-tags "${REMOTE_URL}" "${REMOTE_REF}"; then fail "git fetch ${REMOTE_URL} ${REMOTE_REF} failed" \ "the ref does not exist upstream, or the network is unavailable" \ "check the tag name at ${REMOTE_URL}, or retry with network access" fi -commit="$(git rev-parse "FETCH_HEAD^{commit}")" +commit="$(upstream rev-parse "FETCH_HEAD^{commit}")" # A tag that resolves to a different commit than the pin means upstream moved # it; never follow that silently. @@ -97,13 +106,13 @@ if [[ "${REMOTE_REF}" =~ ^[0-9a-f]{40}$ ]]; then fi fi -upstream_tree="$(git rev-parse "${commit}:mlkem")" -upstream_license="$(git rev-parse "${commit}:LICENSE")" +upstream_tree="$(upstream rev-parse "${commit}:mlkem")" +upstream_license="$(upstream rev-parse "${commit}:LICENSE")" echo "Replacing ${PREFIX} with ${commit}:mlkem and ${commit}:LICENSE" rm -rf -- "${PREFIX}" mkdir -p -- "${PREFIX}" -git archive --format=tar "${commit}" mlkem LICENSE | tar -xf - -C "${PREFIX}" +upstream archive --format=tar "${commit}" mlkem LICENSE | tar -xf - -C "${PREFIX}" cat > "${PIN_FILE}" < Date: Tue, 6 Oct 2026 12:56:17 -0400 Subject: [PATCH 18/20] doc: record the local x86_64 GNU property evidence for mlkem-native #190 asks for ELF property evidence that linking the x86_64 assembly keeps the CET marking. Record what a local Ubuntu build shows: the assembly and C objects and the test_qbit and fuzz binaries that link the native backend all carry IBT and SHSTK, the binaries need only baseline x86-64, and the assembly loses its note only when built with -fcf-protection=none. The Guix release evidence is still outstanding and is required before the post-quantum feature branch merges into 1.x.x. Co-Authored-By: Claude Opus 5.5 --- doc/subtrees/mlkem-native.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/subtrees/mlkem-native.md b/doc/subtrees/mlkem-native.md index 6c4cbd8888..a9e1d07a08 100644 --- a/doc/subtrees/mlkem-native.md +++ b/doc/subtrees/mlkem-native.md @@ -160,6 +160,30 @@ aarch64 and, if no marking is lost, drop the `aarch64-elf` rule in marking by force-marking objects that lack landing pads. macOS arm64 (Mach-O, no GNU property notes) keeps the native backend. +### x86_64 CET marking + +The x86_64 assembly keeps a binary's Intel CET marking. Upstream's `sys.h` +includes `` in the assembly whenever the compiler defines `__CET__`, so +with `-fcf-protection=full`, which qbit's hardening flags pass to the assembly +as to the C (`ci/checks/test_mlkem_build_policy.py` checks that the C flags +reach it), the object gets `endbr64` landing pads and the GNU property note. +Measured for the initial vendoring (v2.0.0) with Ubuntu's gcc 15.2.0 and GNU ld +2.46, `RelWithDebInfo` with qbit's default hardening, native x86_64 enabled: + +```text +mlkem_native_asm.S.o, mlkem_native.c.o, x86 feature: IBT, SHSTK +mlkem_backend.c.o, mlkem.cpp.o +test_qbit, fuzz (both link the native x86 feature: IBT, SHSTK +backend) x86 ISA needed: x86-64-baseline +mlkem_native_asm.S.o assembled with (no x86 feature note) +-fcf-protection=none +``` + +The release evidence is still outstanding for the initial vendoring: Guix +builds of every release host, reproduced, with this comparison for `qbitd`, +`qbit-cli`, `qbit-qt` and `test_qbit` (see the PR checklist below). It is +required before the post-quantum feature branch (#184) merges into `1.x.x`. + ### The AArch64 single-lane Keccak mirror Upstream's `fips202/native/aarch64/x1_scalar.h` runs its scalar assembly From adb75e6f70f622795b31fa48199df0ff1c2732dd Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 13:05:33 -0400 Subject: [PATCH 19/20] test: name the ML-KEM entry point instead of capturing a binding every_entry_point_takes_the_override captured a structured binding in the thread's lambda, which compilers accept only from C++20 (P1091R3, P1381R1). Bind the label and the entry point to ordinary variables, so the test does not depend on that support. Co-Authored-By: Claude Opus 5.5 --- src/test/mlkem_tests.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/mlkem_tests.cpp b/src/test/mlkem_tests.cpp index 546cdb5473..d04e2ce8dd 100644 --- a/src/test/mlkem_tests.cpp +++ b/src/test/mlkem_tests.cpp @@ -597,7 +597,9 @@ BOOST_AUTO_TEST_CASE(every_entry_point_takes_the_override) {"Decaps", [&] { SharedSecret s; (void)Decaps(dk, ct, s); }}, {"GetBackendNames", [] { (void)GetBackendNames(); }}, }; - for (const auto& [name, entry_point] : entry_points) { + for (const auto& entry : entry_points) { + const std::string_view name{entry.first}; + const std::function& entry_point{entry.second}; bool native_while_forced{true}; bool native_after{!native_possible}; std::thread{[&] { From 6f67f86b32b6d3f4f7a8a3163c849e9b97d2c33b Mon Sep 17 00:00:00 2001 From: Dan Hepworth Date: Tue, 6 Oct 2026 13:51:38 -0400 Subject: [PATCH 20/20] ci: build the macOS PQ jobs' Boost and libevent through depends The first macOS arm64 run failed to compile txmempool.h: with NO_DEPENDS=1 the job used Homebrew's Boost, now 1.92, and Boost 1.91 stopped accepting a struct derived from multi_index::indexed_by as an index list. Bitcoin Core fixed the source in bitcoin/bitcoin#35175; that backport is handled on its own. Build Boost and libevent with depends, natively for the runner, so the jobs compile against the versions qbit releases with. Homebrew now installs GNU Make 4 for depends instead of the two libraries. A step sets HOST and reads depends' own build id (packages, recipes and compilers), which keys an actions/cache of depends/built. It is saved once the toolchain file exists, even if the suites fail. After the build, a step requires that depends built that id and that CMake took the toolchain, libevent and Boost at boost.mk's version from it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-pq.yml | 83 ++++++++++++- ci/checks/test_ci_pq_contract.py | 165 +++++++++++++++++++++++++- ci/test/00_setup_env_mac_native_pq.sh | 6 +- 3 files changed, 251 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pq.yml b/.github/workflows/ci-pq.yml index 20c63fc8da..aaa17e9524 100644 --- a/.github/workflows/ci-pq.yml +++ b/.github/workflows/ci-pq.yml @@ -401,14 +401,91 @@ jobs: shell: bash run: echo "BASE_BUILD_DIR=${RUNNER_TEMP}/build" >> "$GITHUB_ENV" + # Boost and libevent come from depends (see FILE_ENV), not Homebrew. + # depends needs GNU Make 4, which Homebrew installs as gmake; its gnubin + # directory makes it the make that 03_test_script.sh runs. - &INSTALL_HOMEBREW_PACKAGES name: Install Homebrew packages env: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 - run: brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent + run: | + brew install --quiet coreutils ninja pkgconf gnu-getopt ccache make + echo "$(brew --prefix make)/libexec/gnubin" >> "$GITHUB_PATH" + + # 03_test_script.sh builds depends for HOST. Set HOST once here, and ask + # depends for the id of that build (its packages, recipes and compilers), + # so the cache key names exactly what the CI script builds. + - &IDENTIFY_DEPENDS + name: Identify depends build + shell: bash + run: | + set -euo pipefail + make_version="$(make --version | head -n 1)" + case "$make_version" in + "GNU Make "[4-9].*) echo "make: $make_version" ;; + *) echo "::error::depends needs GNU Make 4 or later, but make is '$make_version'" >&2; exit 1 ;; + esac + host="$("${BASE_ROOT_DIR}/depends/config.guess")" + # shellcheck source=/dev/null + source "$FILE_ENV" + # As in 03_test_script.sh, bash -c splits DEP_OPTS into make arguments. + id="$(bash -c "make -s -C '${BASE_ROOT_DIR}/depends' HOST=$host $DEP_OPTS print-final_build_id")" + id="${id#final_build_id=}" + printf '%s\n' "$id" | LC_ALL=C grep -Eq '^[0-9a-f]+$' \ + || { echo "::error::depends gave no build id for $host: '$id'" >&2; exit 1; } + echo "depends build $id for $host" + { + echo "HOST=$host" + echo "DEPENDS_BUILD_ID=$id" + } >> "$GITHUB_ENV" + + # Only the built packages are cached; depends rebuilds any whose id + # does not match. + - &RESTORE_DEPENDS + name: Restore depends cache + id: depends-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: depends/built + key: depends-${{ env.HOST }}-${{ env.DEPENDS_BUILD_ID }} - *CI_SCRIPT + # Saved even when the suites fail, but only once depends wrote the + # toolchain file, which needs every package built. + - &SAVE_DEPENDS + name: Save depends cache + if: ${{ !cancelled() && steps.depends-cache.outputs.cache-hit != 'true' && hashFiles(format('depends/{0}/toolchain.cmake', env.HOST)) != '' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: depends/built + key: ${{ steps.depends-cache.outputs.cache-primary-key }} + + # Which libraries did the suites run against? Require the depends build + # the cache key named, its toolchain, its libevent, and Boost at the + # version depends/packages/boost.mk pins. + - &REQUIRE_DEPENDS_BUILD + name: Require depends build + shell: bash + run: | + set -euo pipefail + fail() { echo "::error::$*" >&2; exit 1; } + prefix="${BASE_ROOT_DIR}/depends/${HOST}" + [ -f "${prefix}/.stamp_${DEPENDS_BUILD_ID}" ] \ + || fail "depends did not build ${DEPENDS_BUILD_ID} for ${HOST}, the build the cache key names" + boost_version="$(sed -n 's/^[$](package)_version *= *//p' "${BASE_ROOT_DIR}/depends/packages/boost.mk")" + [ -n "$boost_version" ] || fail "no Boost version in depends/packages/boost.mk" + # Compares the value of a CMakeCache.txt entry, whatever its type. + require_cache_entry() { + local actual + actual="$(sed -n "s/^$1:[A-Z]*=//p" "${BASE_BUILD_DIR}/CMakeCache.txt")" + [ "$actual" = "$2" ] || fail "$1 must be $2, but the build has: ${actual:-$1 unset}" + } + require_cache_entry CMAKE_TOOLCHAIN_FILE "${prefix}/toolchain.cmake" + require_cache_entry Boost_DIR "${prefix}/lib/cmake/Boost-${boost_version}" + require_cache_entry Libevent_DIR "${prefix}/lib/cmake/libevent" + echo "Built with Boost ${boost_version} and libevent from ${prefix}" + # The suites passed, but on which code? A runner whose CPU lacks an # extension the native code needs would pass them on portable C, so # require the native backend by name. mlkem_tests/backend_selection logs @@ -463,6 +540,10 @@ jobs: - *REQUIRE_MACHINE - *SET_BUILD_DIR - *INSTALL_HOMEBREW_PACKAGES + - *IDENTIFY_DEPENDS + - *RESTORE_DEPENDS - *CI_SCRIPT + - *SAVE_DEPENDS + - *REQUIRE_DEPENDS_BUILD - *REQUIRE_NATIVE_MLKEM - *MACOS_REPORT_EVIDENCE diff --git a/ci/checks/test_ci_pq_contract.py b/ci/checks/test_ci_pq_contract.py index e0fff0d788..bc8a7f4315 100755 --- a/ci/checks/test_ci_pq_contract.py +++ b/ci/checks/test_ci_pq_contract.py @@ -37,12 +37,14 @@ import json import os import re +import shutil import subprocess import sys import tempfile import unittest from pathlib import Path from typing import Any +from unittest import mock import ctest_evidence import test_scheduled_validation_contract as svc @@ -57,6 +59,7 @@ GATE_WORKFLOW = svc.WORKFLOW_DIR / "required-merge-gate.yml" TEST_SCRIPT = REPO_ROOT / "ci" / "test" / "03_test_script.sh" MLKEM_TESTS = REPO_ROOT / "src" / "test" / "mlkem_tests.cpp" +BOOST_RECIPE = REPO_ROOT / "depends" / "packages" / "boost.mk" MLKEM_BACKEND_HEADERS = [REPO_ROOT / "src" / "crypto" / name for name in ("mlkem_arith_backend.h", "mlkem_fips202_backend.h")] RESOLVER_JOB = svc.RESOLVER_JOB TRIGGER_STEP = "Check how the run started" @@ -64,6 +67,11 @@ SHARED_RESOLVER = "resolve_source_ref" SELECT_STEP = "Select jobs" REPORT_STEP = "Report unit-test evidence" +DEPENDS_INSTALL_STEP = "Install Homebrew packages" +DEPENDS_IDENTIFY_STEP = "Identify depends build" +DEPENDS_RESTORE_STEP = "Restore depends cache" +DEPENDS_SAVE_STEP = "Save depends cache" +DEPENDS_REQUIRE_STEP = "Require depends build" PQ_UNIT_JOB = "unit" AARCH64_JOB = "aarch64-unit" S390X_JOB = "s390x-unit" @@ -187,6 +195,14 @@ def source_env(env_file: str, names: list[str], extra_env: dict[str, str] | None return svc.parse_key_values(completed.stdout) +def depends_print(host: str, dep_opts: str, *names: str) -> dict[str, str]: + """Values depends/Makefile computes for host with dep_opts, read with its print- rule (nothing is built).""" + targets = " ".join(f"print-{name}" for name in names) + completed = subprocess.run(["bash", "-c", f"make -s -C depends HOST={host} {dep_opts} {targets}"], cwd=REPO_ROOT, + text=True, capture_output=True, check=True) + return {name: value.strip() for name, value in svc.parse_key_values(completed.stdout).items()} + + def shell_block(text: str, start: str, end_line: str) -> str: """The lines of text from the line starting with start through the next line equal to end_line.""" lines = text.splitlines() @@ -940,7 +956,7 @@ def check_macos_job(self, workflow: dict[str, Any], job_id: str, runner: str, ma # Losing the native backend must fail the configure step, not fall back quietly. self.assertIn("-DWITH_MLKEM_NATIVE=ON", effective["BITCOIN_CONFIG"].split()) self.assertEqual(effective["CI_OS_NAME"], "macos") - self.assertEqual(effective["NO_DEPENDS"], "1") + self.assertEqual(effective["NO_DEPENDS"], "", "Boost and libevent come from depends, not Homebrew") self.assertEqual(effective["CI_BUILD_TARGET"], "test_bitcoin") self.assertEqual(effective["CTEST_EXPECTED_SUITES"].split(), PQ_SUITES) self.assertEqual(effective["RUN_FUNCTIONAL_TESTS"], "false") @@ -1028,6 +1044,153 @@ def check_macos_job(self, workflow: dict[str, Any], job_id: str, runner: str, ma self.assertIn(evidence_word, result.summary) self.assertIn("| mlkem_tests | passed | 0.5 |", result.summary) + def test_macos_jobs_build_boost_and_libevent_through_depends(self) -> None: + workflow = self.workflows[PQ] + for job_id, (_runner, _machine, arch, _evidence_word) in MACOS_JOBS.items(): + with self.subTest(job=job_id): + self.check_macos_depends(workflow, job_id, f"{arch}-apple-darwin24.6.0") + + def check_macos_depends(self, workflow: dict[str, Any], job_id: str, runner_host: str) -> None: + fixture = self.fixture + job = workflow["jobs"][job_id] + context = self.resolve(PQ, self.dispatch(PQ, jobs=job_id)) + github_env = svc.github_env_for(context, job_id, workflow["name"]) + env_file = svc.job_env(workflow, job, context)["FILE_ENV"] + dep_opts = source_env(env_file, ["DEP_OPTS"])["DEP_OPTS"] + + # For the runner's host, depends builds Boost and libevent and nothing else. + printed = depends_print(runner_host, dep_opts, "packages", "native_packages") + self.assertEqual(printed["packages"].split(), ["boost", "libevent"]) + self.assertEqual(printed["native_packages"].split(), []) + + order = [DEPENDS_INSTALL_STEP, DEPENDS_IDENTIFY_STEP, DEPENDS_RESTORE_STEP, "CI script", DEPENDS_SAVE_STEP, + DEPENDS_REQUIRE_STEP, "Require native ML-KEM", REPORT_STEP] + self.assertEqual([svc.step_index(job, name) for name in order], sorted(svc.step_index(job, name) for name in order)) + + # Homebrew provides GNU Make 4 as the make on PATH, but neither library. + step = svc.find_step(job, DEPENDS_INSTALL_STEP) + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + stubs = Path(tmp) + (stubs / "brew").write_text('#!/bin/sh\nif [ "$1" = --prefix ]; then echo "/stub/opt/$2"; ' + 'else echo "$*" >> "$STUB_LOG"; fi\n', encoding="utf8") + (stubs / "brew").chmod(0o755) + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"PATH": f"{stubs}:{os.environ['PATH']}", "STUB_LOG": str(stubs / "log"), + "GITHUB_PATH": str(stubs / "github-path")}) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode, 0, result.stderr) + installed = (stubs / "log").read_text(encoding="utf8").split() + self.assertEqual(installed[:2], ["install", "--quiet"]) + self.assertIn("make", installed) + self.assertNotIn("boost", installed) + self.assertNotIn("libevent", installed) + self.assertEqual((stubs / "github-path").read_text(encoding="utf8"), "/stub/opt/make/libexec/gnubin\n") + + # HOST and the cache key's id are what depends reports for this build. + step = svc.find_step(job, DEPENDS_IDENTIFY_STEP) + host = subprocess.run([str(REPO_ROOT / "depends" / "config.guess")], text=True, capture_output=True, + check=True).stdout.strip() + expected_id = depends_print(host, dep_opts, "final_build_id")["final_build_id"] + self.assertRegex(expected_id, r"^[0-9a-f]+$") + with tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + # A working make that reports macOS's own GNU Make 3.81. + stubs = Path(tmp) / "stubs" + stubs.mkdir() + (stubs / "make").write_text("#!/bin/sh\n" + "if [ \"$1\" = --version ]; then echo 'GNU Make 3.81'; exit 0; fi\n" + f"exec {shutil.which('make')} \"$@\"\n", encoding="utf8") + (stubs / "make").chmod(0o755) + for label, path, ok in (("GNU Make 4", os.environ["PATH"], True), ("GNU Make 3.81", f"{stubs}:{os.environ['PATH']}", False)): + with self.subTest(job=job_id, make=label): + github_env_file = Path(tmp) / f"github-env-{ok}" + github_env_file.touch() + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"PATH": path, "BASE_ROOT_DIR": str(REPO_ROOT), "GITHUB_ENV": str(github_env_file)}) + result = svc.run_step(fixture, step, step_env, REPO_ROOT, github_env) + self.assertEqual(result.returncode == 0, ok, result.completed.stdout + result.stderr) + if ok: + self.assertEqual(github_env_file.read_text(encoding="utf8"), f"HOST={host}\nDEPENDS_BUILD_ID={expected_id}\n") + else: + self.assertIn("::error::depends needs GNU Make 4 or later", result.stderr) + self.assertEqual(github_env_file.read_text(encoding="utf8"), "") + + # The cache holds depends/built under that key, and is saved under the + # same key once the CI script wrote the toolchain file. + restore = svc.find_step(job, DEPENDS_RESTORE_STEP) + save = svc.find_step(job, DEPENDS_SAVE_STEP) + self.assertEqual(restore["uses"].split("@"), ["actions/cache/restore", save["uses"].split("@")[1]]) + self.assertEqual(save["uses"].split("@")[0], "actions/cache/save") + self.assertEqual(restore["with"]["path"], "depends/built") + self.assertEqual(save["with"]["path"], restore["with"]["path"]) + self.assertEqual(save["with"]["key"], "${{ steps." + restore["id"] + ".outputs.cache-primary-key }}") + step_context = dict(context, env={"HOST": runner_host, "DEPENDS_BUILD_ID": "0123456789a"}) + self.assertEqual(svc.render(restore["with"]["key"], step_context), f"depends-{runner_host}-0123456789a") + toolchain = f"depends/{runner_host}/toolchain.cmake" + save_cases = [ + ("built on a miss", "", [toolchain], False, True), + ("restore-key match", "false", [toolchain], False, True), + ("exact hit", "true", [toolchain], False, False), + ("depends did not finish", "", [], False, False), + ("cancelled", "", [toolchain], True, False), + ] + for label, cache_hit, files, cancelled, saves in save_cases: + with self.subTest(job=job_id, save=label): + functions = {"cancelled": lambda cancelled=cancelled: cancelled, + "hashfiles": lambda pattern, files=files: "f00d" if pattern in files else ""} + save_context = dict(step_context, steps={restore["id"]: {"outputs": {"cache-hit": cache_hit}}}) + with mock.patch.dict(svc._FUNCTIONS, functions): + self.assertIs(svc._truthy(svc.render(save["if"], save_context)), saves) + + # Afterwards the build must have used that depends tree: its toolchain, + # libevent, and Boost at the version boost.mk pins. + step = svc.find_step(job, DEPENDS_REQUIRE_STEP) + pinned = re.search(r"^\$\(package\)_version *= *(\S+)$", BOOST_RECIPE.read_text(encoding="utf8"), re.M) + assert pinned is not None, BOOST_RECIPE + boost_version = pinned.group(1) + build_id = "0123456789a" + build_cases: list[tuple[str, dict[str, str | None], str, str | None]] = [ + ("depends", {}, build_id, None), + ("depends, other entry type", {"CMAKE_TOOLCHAIN_FILE:FILEPATH": None, + "CMAKE_TOOLCHAIN_FILE:UNINITIALIZED": "{prefix}/toolchain.cmake"}, build_id, None), + ("stale cache key", {}, "fedcba98765", "depends did not build 0123456789a"), + ("no depends", {"CMAKE_TOOLCHAIN_FILE:FILEPATH": None, + "Boost_DIR:PATH": "/opt/homebrew/Cellar/boost/1.92.0/lib/cmake/Boost-1.92.0"}, + build_id, "CMAKE_TOOLCHAIN_FILE unset"), + ("Homebrew Boost", {"Boost_DIR:PATH": "/opt/homebrew/Cellar/boost/1.92.0/lib/cmake/Boost-1.92.0"}, + build_id, "Boost_DIR must be {prefix}/lib/cmake/Boost-" + boost_version), + ("other depends Boost", {"Boost_DIR:PATH": "{prefix}/lib/cmake/Boost-1.0.0"}, build_id, "Boost-1.0.0"), + ("Homebrew libevent", {"Libevent_DIR:PATH": "/opt/homebrew/lib/cmake/libevent"}, build_id, "Libevent_DIR must be"), + ] + for label, changes, stamp_id, error in build_cases: + with self.subTest(job=job_id, depends_build=label), tempfile.TemporaryDirectory(dir=fixture.root) as tmp: + root = Path(tmp) / "workspace" + prefix = root / "depends" / runner_host + prefix.mkdir(parents=True) + (prefix / f".stamp_{stamp_id}").touch() + (root / "depends" / "packages").mkdir() + shutil.copy(BOOST_RECIPE, root / "depends" / "packages") + entries: dict[str, str | None] = { + "CMAKE_TOOLCHAIN_FILE:FILEPATH": "{prefix}/toolchain.cmake", + "Boost_DIR:PATH": "{prefix}/lib/cmake/Boost-" + boost_version, + "Libevent_DIR:PATH": "{prefix}/lib/cmake/libevent", + } + entries.update(changes) + build = Path(tmp) / "build" + build.mkdir() + (build / "CMakeCache.txt").write_text( + "".join(f"{key}={value.format(prefix=prefix)}\n" for key, value in entries.items() if value is not None), + encoding="utf8") + step_env = svc.step_env(workflow, job, step, context) + step_env.update({"BASE_ROOT_DIR": str(root), "BASE_BUILD_DIR": str(build), "HOST": runner_host, + "DEPENDS_BUILD_ID": build_id}) + result = svc.run_step(fixture, step, step_env, fixture.root, github_env) + self.assertEqual(result.returncode == 0, error is None, result.completed.stdout + result.stderr) + if error is None: + self.assertIn(f"Built with Boost {boost_version} and libevent from {prefix}", result.completed.stdout) + else: + self.assertIn("::error::", result.stderr) + self.assertIn(error.format(prefix=prefix), result.stderr) + def test_contexts_are_available_where_used(self) -> None: """Mirror actionlint's context-availability rule for the places these workflows use expressions. diff --git a/ci/test/00_setup_env_mac_native_pq.sh b/ci/test/00_setup_env_mac_native_pq.sh index 39b6601fa5..f2777eebea 100755 --- a/ci/test/00_setup_env_mac_native_pq.sh +++ b/ci/test/00_setup_env_mac_native_pq.sh @@ -13,7 +13,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME="ci_mac_native_pq" # macos does not use a container, but the env var is needed for logging export CMAKE_GENERATOR="Ninja" export CI_OS_NAME="macos" -export NO_DEPENDS=1 +# Boost and libevent come from depends, built natively for the runner, so the +# jobs compile against the versions qbit releases with. Homebrew's Boost moves +# on its own: 1.92 stopped compiling the multi_index declarations in +# txmempool.h (fixed upstream by bitcoin/bitcoin#35175). +export DEP_OPTS="NO_QT=1 NO_QR=1 NO_ZMQ=1 NO_WALLET=1 NO_USDT=1 NO_IPC=1" export OSX_SDK="" export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false